diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
85 files changed, 24298 insertions, 14210 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 74acca9bcd9d..fcd77b27514d 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -10,6 +10,7 @@ config DRM_I915 # the shmem_readpage() which depends upon tmpfs select SHMEM select TMPFS + select STOP_MACHINE select DRM_KMS_HELPER select DRM_PANEL select DRM_MIPI_DSI @@ -36,30 +37,6 @@ config DRM_I915 i810 driver instead, and the Atom z5xx series has an entirely different implementation. -config DRM_I915_KMS - bool "Enable modesetting on intel by default" - depends on DRM_I915 - default y - help - Choose this option if you want kernel modesetting enabled by default. - - If in doubt, say "Y". - -config DRM_I915_FBDEV - bool "Enable legacy fbdev support for the modesetting intel driver" - depends on DRM_I915 - select DRM_KMS_FB_HELPER - select FB_CFB_FILLRECT - select FB_CFB_COPYAREA - select FB_CFB_IMAGEBLIT - default y - help - Choose this option if you have a need for the legacy fbdev - support. Note that this support also provide the linux console - support on top of the intel modesetting driver. - - If in doubt, say "Y". - config DRM_I915_PRELIMINARY_HW_SUPPORT bool "Enable preliminary support for prerelease Intel hardware by default" depends on DRM_I915 diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index b7ddf48e1d75..0851de07bd13 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -6,12 +6,13 @@ # core driver code i915-y := i915_drv.o \ + i915_irq.o \ i915_params.o \ i915_suspend.o \ i915_sysfs.o \ + intel_csr.o \ intel_pm.o \ - intel_runtime_pm.o \ - intel_csr.o + intel_runtime_pm.o i915-$(CONFIG_COMPAT) += i915_ioc32.o i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o @@ -20,24 +21,29 @@ i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o i915-y += i915_cmd_parser.o \ i915_gem_batch_pool.o \ i915_gem_context.o \ - i915_gem_render_state.o \ i915_gem_debug.o \ i915_gem_dmabuf.o \ i915_gem_evict.o \ i915_gem_execbuffer.o \ + i915_gem_fence.o \ i915_gem_gtt.o \ i915_gem.o \ + i915_gem_render_state.o \ i915_gem_shrinker.o \ i915_gem_stolen.o \ i915_gem_tiling.o \ i915_gem_userptr.o \ i915_gpu_error.o \ - i915_irq.o \ i915_trace_points.o \ intel_lrc.o \ + intel_mocs.o \ intel_ringbuffer.o \ intel_uncore.o +# general-purpose microcontroller (GuC) support +i915-y += intel_guc_loader.o \ + i915_guc_submission.o + # autogenerated null render state i915-y += intel_renderstate_gen6.o \ intel_renderstate_gen7.o \ @@ -46,18 +52,21 @@ i915-y += intel_renderstate_gen6.o \ # modesetting core code i915-y += intel_audio.o \ + intel_atomic.o \ + intel_atomic_plane.o \ intel_bios.o \ intel_display.o \ intel_fbc.o \ intel_fifo_underrun.o \ intel_frontbuffer.o \ + intel_hotplug.o \ intel_modes.o \ intel_overlay.o \ intel_psr.o \ intel_sideband.o \ intel_sprite.o i915-$(CONFIG_ACPI) += intel_acpi.o intel_opregion.o -i915-$(CONFIG_DRM_I915_FBDEV) += intel_fbdev.o +i915-$(CONFIG_DRM_FBDEV_EMULATION) += intel_fbdev.o # modesetting output/encoder code i915-y += dvo_ch7017.o \ @@ -66,15 +75,14 @@ i915-y += dvo_ch7017.o \ dvo_ns2501.o \ dvo_sil164.o \ dvo_tfp410.o \ - intel_atomic.o \ - intel_atomic_plane.o \ intel_crt.o \ intel_ddi.o \ - intel_dp.o \ + intel_dp_link_training.o \ intel_dp_mst.o \ + intel_dp.o \ intel_dsi.o \ - intel_dsi_pll.o \ intel_dsi_panel_vbt.o \ + intel_dsi_pll.o \ intel_dvo.o \ intel_hdmi.o \ intel_i2c.o \ diff --git a/drivers/gpu/drm/i915/dvo.h b/drivers/gpu/drm/i915/dvo.h index 312163379db9..13dea4263554 100644 --- a/drivers/gpu/drm/i915/dvo.h +++ b/drivers/gpu/drm/i915/dvo.h @@ -32,7 +32,8 @@ struct intel_dvo_device { const char *name; int type; /* DVOA/B/C output register */ - u32 dvo_reg; + i915_reg_t dvo_reg; + i915_reg_t dvo_srcdim_reg; /* GPIO register used for i2c bus to control this device */ u32 gpio; int slave_addr; @@ -94,8 +95,8 @@ struct intel_dvo_dev_ops { * after this function is called. */ void (*mode_set)(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode); + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode); /* * Probe for a connected output, and return detect_status. diff --git a/drivers/gpu/drm/i915/dvo_ch7017.c b/drivers/gpu/drm/i915/dvo_ch7017.c index 86b27d1d90c2..cbb22027a3ce 100644 --- a/drivers/gpu/drm/i915/dvo_ch7017.c +++ b/drivers/gpu/drm/i915/dvo_ch7017.c @@ -255,8 +255,8 @@ static enum drm_mode_status ch7017_mode_valid(struct intel_dvo_device *dvo, } static void ch7017_mode_set(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode) { uint8_t lvds_pll_feedback_div, lvds_pll_vco_control; uint8_t outputs_enable, lvds_control_2, lvds_power_down; diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c index 80449f475960..4b4acc1a06fe 100644 --- a/drivers/gpu/drm/i915/dvo_ch7xxx.c +++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c @@ -275,8 +275,8 @@ static enum drm_mode_status ch7xxx_mode_valid(struct intel_dvo_device *dvo, } static void ch7xxx_mode_set(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode) { uint8_t tvco, tpcp, tpd, tlpf, idf; diff --git a/drivers/gpu/drm/i915/dvo_ivch.c b/drivers/gpu/drm/i915/dvo_ivch.c index 89b08a896d20..ff9f1b077d83 100644 --- a/drivers/gpu/drm/i915/dvo_ivch.c +++ b/drivers/gpu/drm/i915/dvo_ivch.c @@ -22,6 +22,7 @@ * * Authors: * Eric Anholt <eric@anholt.net> + * Thomas Richter <thor@math.tu-berlin.de> * * Minor modifications (Dithering enable): * Thomas Richter <thor@math.tu-berlin.de> @@ -90,7 +91,7 @@ /* * LCD Vertical Display Size */ -#define VR21 0x20 +#define VR21 0x21 /* * Panel power down status @@ -155,16 +156,33 @@ # define VR8F_POWER_MASK (0x3c) # define VR8F_POWER_POS (2) +/* Some Bios implementations do not restore the DVO state upon + * resume from standby. Thus, this driver has to handle it + * instead. The following list contains all registers that + * require saving. + */ +static const uint16_t backup_addresses[] = { + 0x11, 0x12, + 0x18, 0x19, 0x1a, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x8e, 0x8f, + 0x10 /* this must come last */ +}; + struct ivch_priv { bool quiet; uint16_t width, height; + + /* Register backup */ + + uint16_t reg_backup[ARRAY_SIZE(backup_addresses)]; }; static void ivch_dump_regs(struct intel_dvo_device *dvo); - /** * Reads a register on the ivch. * @@ -246,6 +264,7 @@ static bool ivch_init(struct intel_dvo_device *dvo, { struct ivch_priv *priv; uint16_t temp; + int i; priv = kzalloc(sizeof(struct ivch_priv), GFP_KERNEL); if (priv == NULL) @@ -273,6 +292,14 @@ static bool ivch_init(struct intel_dvo_device *dvo, ivch_read(dvo, VR20, &priv->width); ivch_read(dvo, VR21, &priv->height); + /* Make a backup of the registers to be able to restore them + * upon suspend. + */ + for (i = 0; i < ARRAY_SIZE(backup_addresses); i++) + ivch_read(dvo, backup_addresses[i], priv->reg_backup + i); + + ivch_dump_regs(dvo); + return true; out: @@ -294,12 +321,31 @@ static enum drm_mode_status ivch_mode_valid(struct intel_dvo_device *dvo, return MODE_OK; } +/* Restore the DVO registers after a resume + * from RAM. Registers have been saved during + * the initialization. + */ +static void ivch_reset(struct intel_dvo_device *dvo) +{ + struct ivch_priv *priv = dvo->dev_priv; + int i; + + DRM_DEBUG_KMS("Resetting the IVCH registers\n"); + + ivch_write(dvo, VR10, 0x0000); + + for (i = 0; i < ARRAY_SIZE(backup_addresses); i++) + ivch_write(dvo, backup_addresses[i], priv->reg_backup[i]); +} + /** Sets the power state of the panel connected to the ivch */ static void ivch_dpms(struct intel_dvo_device *dvo, bool enable) { int i; uint16_t vr01, vr30, backlight; + ivch_reset(dvo); + /* Set the new power state of the panel. */ if (!ivch_read(dvo, VR01, &vr01)) return; @@ -308,6 +354,7 @@ static void ivch_dpms(struct intel_dvo_device *dvo, bool enable) backlight = 1; else backlight = 0; + ivch_write(dvo, VR80, backlight); if (enable) @@ -334,6 +381,8 @@ static bool ivch_get_hw_state(struct intel_dvo_device *dvo) { uint16_t vr01; + ivch_reset(dvo); + /* Set the new power state of the panel. */ if (!ivch_read(dvo, VR01, &vr01)) return false; @@ -345,14 +394,18 @@ static bool ivch_get_hw_state(struct intel_dvo_device *dvo) } static void ivch_mode_set(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode) { + struct ivch_priv *priv = dvo->dev_priv; uint16_t vr40 = 0; uint16_t vr01 = 0; uint16_t vr10; - ivch_read(dvo, VR10, &vr10); + ivch_reset(dvo); + + vr10 = priv->reg_backup[ARRAY_SIZE(backup_addresses) - 1]; + /* Enable dithering for 18 bpp pipelines */ vr10 &= VR10_INTERFACE_DEPTH_MASK; if (vr10 == VR10_INTERFACE_2X18 || vr10 == VR10_INTERFACE_1X18) @@ -361,16 +414,16 @@ static void ivch_mode_set(struct intel_dvo_device *dvo, vr40 = (VR40_STALL_ENABLE | VR40_VERTICAL_INTERP_ENABLE | VR40_HORIZONTAL_INTERP_ENABLE); - if (mode->hdisplay != adjusted_mode->hdisplay || - mode->vdisplay != adjusted_mode->vdisplay) { + if (mode->hdisplay != adjusted_mode->crtc_hdisplay || + mode->vdisplay != adjusted_mode->crtc_vdisplay) { uint16_t x_ratio, y_ratio; vr01 |= VR01_PANEL_FIT_ENABLE; - vr40 |= VR40_CLOCK_GATING_ENABLE | VR40_ENHANCED_PANEL_FITTING; + vr40 |= VR40_CLOCK_GATING_ENABLE; x_ratio = (((mode->hdisplay - 1) << 16) / - (adjusted_mode->hdisplay - 1)) >> 2; + (adjusted_mode->crtc_hdisplay - 1)) >> 2; y_ratio = (((mode->vdisplay - 1) << 16) / - (adjusted_mode->vdisplay - 1)) >> 2; + (adjusted_mode->crtc_vdisplay - 1)) >> 2; ivch_write(dvo, VR42, x_ratio); ivch_write(dvo, VR41, y_ratio); } else { @@ -381,8 +434,6 @@ static void ivch_mode_set(struct intel_dvo_device *dvo, ivch_write(dvo, VR01, vr01); ivch_write(dvo, VR40, vr40); - - ivch_dump_regs(dvo); } static void ivch_dump_regs(struct intel_dvo_device *dvo) diff --git a/drivers/gpu/drm/i915/dvo_ns2501.c b/drivers/gpu/drm/i915/dvo_ns2501.c index 97ae8aa157e9..063859fff0f0 100644 --- a/drivers/gpu/drm/i915/dvo_ns2501.c +++ b/drivers/gpu/drm/i915/dvo_ns2501.c @@ -546,8 +546,8 @@ static enum drm_mode_status ns2501_mode_valid(struct intel_dvo_device *dvo, } static void ns2501_mode_set(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode) { const struct ns2501_configuration *conf; struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv); diff --git a/drivers/gpu/drm/i915/dvo_sil164.c b/drivers/gpu/drm/i915/dvo_sil164.c index fa0114967076..26f13eb634f9 100644 --- a/drivers/gpu/drm/i915/dvo_sil164.c +++ b/drivers/gpu/drm/i915/dvo_sil164.c @@ -190,8 +190,8 @@ static enum drm_mode_status sil164_mode_valid(struct intel_dvo_device *dvo, } static void sil164_mode_set(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode) { /* As long as the basics are set up, since we don't have clock * dependencies in the mode setup, we can just leave the diff --git a/drivers/gpu/drm/i915/dvo_tfp410.c b/drivers/gpu/drm/i915/dvo_tfp410.c index 7853719a0e81..6f1a0a6d4e22 100644 --- a/drivers/gpu/drm/i915/dvo_tfp410.c +++ b/drivers/gpu/drm/i915/dvo_tfp410.c @@ -222,8 +222,8 @@ static enum drm_mode_status tfp410_mode_valid(struct intel_dvo_device *dvo, } static void tfp410_mode_set(struct intel_dvo_device *dvo, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *mode, + const struct drm_display_mode *adjusted_mode) { /* As long as the basics are set up, since we don't have clock dependencies * in the mode setup, we can just leave the registers alone and everything diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index 306d9e4e5cf3..814d894ed925 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -94,7 +94,7 @@ #define CMD(op, opm, f, lm, fl, ...) \ { \ .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \ - .cmd = { (op), (opm) }, \ + .cmd = { (op), (opm) }, \ .length = { (lm) }, \ __VA_ARGS__ \ } @@ -124,14 +124,14 @@ static const struct drm_i915_cmd_descriptor common_cmds[] = { CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ), CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W, .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ), - CMD( MI_STORE_REGISTER_MEM(1), SMI, !F, 0xFF, W | B, + CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B, .reg = { .offset = 1, .mask = 0x007FFFFC }, .bits = {{ .offset = 0, .mask = MI_GLOBAL_GTT, .expected = 0, }}, ), - CMD( MI_LOAD_REGISTER_MEM, SMI, !F, 0xFF, W | B, + CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B, .reg = { .offset = 1, .mask = 0x007FFFFC }, .bits = {{ .offset = 0, @@ -151,8 +151,8 @@ static const struct drm_i915_cmd_descriptor render_cmds[] = { CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), CMD( MI_PREDICATE, SMI, F, 1, S ), CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ), - CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), CMD( MI_SET_APPID, SMI, F, 1, S ), + CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ), CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ), CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B, @@ -407,14 +407,14 @@ static const struct drm_i915_cmd_table hsw_blt_ring_cmds[] = { * LRI. */ struct drm_i915_reg_descriptor { - u32 addr; + i915_reg_t addr; u32 mask; u32 value; }; /* Convenience macro for adding 32-bit registers. */ -#define REG32(address, ...) \ - { .addr = address, __VA_ARGS__ } +#define REG32(_reg, ...) \ + { .addr = (_reg), __VA_ARGS__ } /* * Convenience macro for adding 64-bit registers. @@ -423,8 +423,13 @@ struct drm_i915_reg_descriptor { * access commands only allow 32-bit accesses. Hence, we have to include * entries for both halves of the 64-bit registers. */ -#define REG64(addr) \ - REG32(addr), REG32(addr + sizeof(u32)) +#define REG64(_reg) \ + { .addr = _reg }, \ + { .addr = _reg ## _UDW } + +#define REG64_IDX(_reg, idx) \ + { .addr = _reg(idx) }, \ + { .addr = _reg ## _UDW(idx) } static const struct drm_i915_reg_descriptor gen7_render_regs[] = { REG64(GPGPU_THREADS_DISPATCHED), @@ -448,14 +453,17 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = { REG32(GEN7_3DPRIM_INSTANCE_COUNT), REG32(GEN7_3DPRIM_START_INSTANCE), REG32(GEN7_3DPRIM_BASE_VERTEX), - REG64(GEN7_SO_NUM_PRIMS_WRITTEN(0)), - REG64(GEN7_SO_NUM_PRIMS_WRITTEN(1)), - REG64(GEN7_SO_NUM_PRIMS_WRITTEN(2)), - REG64(GEN7_SO_NUM_PRIMS_WRITTEN(3)), - REG64(GEN7_SO_PRIM_STORAGE_NEEDED(0)), - REG64(GEN7_SO_PRIM_STORAGE_NEEDED(1)), - REG64(GEN7_SO_PRIM_STORAGE_NEEDED(2)), - REG64(GEN7_SO_PRIM_STORAGE_NEEDED(3)), + REG32(GEN7_GPGPU_DISPATCHDIMX), + REG32(GEN7_GPGPU_DISPATCHDIMY), + REG32(GEN7_GPGPU_DISPATCHDIMZ), + REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0), + REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1), + REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2), + REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3), + REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0), + REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1), + REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2), + REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3), REG32(GEN7_SO_WRITE_OFFSET(0)), REG32(GEN7_SO_WRITE_OFFSET(1)), REG32(GEN7_SO_WRITE_OFFSET(2)), @@ -564,7 +572,7 @@ static bool validate_cmds_sorted(struct intel_engine_cs *ring, for (j = 0; j < table->count; j++) { const struct drm_i915_cmd_descriptor *desc = - &table->table[i]; + &table->table[j]; u32 curr = desc->cmd.value & desc->cmd.mask; if (curr < previous) { @@ -589,7 +597,7 @@ static bool check_sorted(int ring_id, bool ret = true; for (i = 0; i < reg_count; i++) { - u32 curr = reg_table[i].addr; + u32 curr = i915_mmio_reg_offset(reg_table[i].addr); if (curr < previous) { DRM_ERROR("CMD: table not sorted ring=%d entry=%d reg=0x%08X prev=0x%08X\n", @@ -844,7 +852,7 @@ find_reg(const struct drm_i915_reg_descriptor *table, int i; for (i = 0; i < count; i++) { - if (table[i].addr == addr) + if (i915_mmio_reg_offset(table[i].addr) == addr) return &table[i]; } } @@ -1020,7 +1028,7 @@ static bool check_cmd(const struct intel_engine_cs *ring, * to the register. Hence, limit OACONTROL writes to * only MI_LOAD_REGISTER_IMM commands. */ - if (reg_addr == OACONTROL) { + if (reg_addr == i915_mmio_reg_offset(OACONTROL)) { if (desc->cmd.value == MI_LOAD_REGISTER_MEM) { DRM_DEBUG_DRIVER("CMD: Rejected LRM to OACONTROL\n"); return false; @@ -1213,6 +1221,8 @@ int i915_cmd_parser_get_version(void) * 2. Allow access to the MI_PREDICATE_SRC0 and * MI_PREDICATE_SRC1 registers. * 3. Allow access to the GPGPU_THREADS_DISPATCHED register. + * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3. + * 5. GPGPU dispatch compute indirect registers. */ - return 3; + return 5; } diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 82bbe3f2a7e1..a8721fccd8a0 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -46,11 +46,6 @@ enum { PINNED_LIST, }; -static const char *yesno(int v) -{ - return v ? "yes" : "no"; -} - /* As the drm_debugfs_init() routines are called before dev->dev_private is * allocated we need to hook into the minor for release. */ static int @@ -117,6 +112,20 @@ static inline const char *get_global_flag(struct drm_i915_gem_object *obj) return i915_gem_obj_to_ggtt(obj) ? "g" : " "; } +static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj) +{ + u64 size = 0; + struct i915_vma *vma; + + list_for_each_entry(vma, &obj->vma_list, vma_link) { + if (i915_is_ggtt(vma->vm) && + drm_mm_node_allocated(&vma->node)) + size += vma->node.size; + } + + return size; +} + static void describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { @@ -156,13 +165,13 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (obj->fence_reg != I915_FENCE_REG_NONE) seq_printf(m, " (fence: %d)", obj->fence_reg); list_for_each_entry(vma, &obj->vma_list, vma_link) { - if (!i915_is_ggtt(vma->vm)) - seq_puts(m, " (pp"); + seq_printf(m, " (%sgtt offset: %08llx, size: %08llx", + i915_is_ggtt(vma->vm) ? "g" : "pp", + vma->node.start, vma->node.size); + if (i915_is_ggtt(vma->vm)) + seq_printf(m, ", type: %u)", vma->ggtt_view.type); else - seq_puts(m, " (g"); - seq_printf(m, "gtt offset: %08llx, size: %08llx, type: %u)", - vma->node.start, vma->node.size, - vma->ggtt_view.type); + seq_puts(m, ")"); } if (obj->stolen) seq_printf(m, " (stolen: %08llx)", obj->stolen->start); @@ -198,7 +207,7 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data) struct drm_i915_private *dev_priv = dev->dev_private; struct i915_address_space *vm = &dev_priv->gtt.base; struct i915_vma *vma; - size_t total_obj_size, total_gtt_size; + u64 total_obj_size, total_gtt_size; int count, ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -231,7 +240,7 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data) } mutex_unlock(&dev->struct_mutex); - seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n", + seq_printf(m, "Total %d objects, %llu bytes, %llu GTT size\n", count, total_obj_size, total_gtt_size); return 0; } @@ -244,7 +253,11 @@ static int obj_rank_by_stolen(void *priv, struct drm_i915_gem_object *b = container_of(B, struct drm_i915_gem_object, obj_exec_link); - return a->stolen->start - b->stolen->start; + if (a->stolen->start < b->stolen->start) + return -1; + if (a->stolen->start > b->stolen->start) + return 1; + return 0; } static int i915_gem_stolen_list_info(struct seq_file *m, void *data) @@ -253,7 +266,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; - size_t total_obj_size, total_gtt_size; + u64 total_obj_size, total_gtt_size; LIST_HEAD(stolen); int count, ret; @@ -269,7 +282,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) list_add(&obj->obj_exec_link, &stolen); total_obj_size += obj->base.size; - total_gtt_size += i915_gem_obj_ggtt_size(obj); + total_gtt_size += i915_gem_obj_total_ggtt_size(obj); count++; } list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { @@ -292,14 +305,14 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) } mutex_unlock(&dev->struct_mutex); - seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n", + seq_printf(m, "Total %d objects, %llu bytes, %llu GTT size\n", count, total_obj_size, total_gtt_size); return 0; } #define count_objects(list, member) do { \ list_for_each_entry(obj, list, member) { \ - size += i915_gem_obj_ggtt_size(obj); \ + size += i915_gem_obj_total_ggtt_size(obj); \ ++count; \ if (obj->map_and_fenceable) { \ mappable_size += i915_gem_obj_ggtt_size(obj); \ @@ -310,10 +323,10 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) struct file_stats { struct drm_i915_file_private *file_priv; - int count; - size_t total, unbound; - size_t global, shared; - size_t active, inactive; + unsigned long count; + u64 total, unbound; + u64 global, shared; + u64 active, inactive; }; static int per_file_stats(int id, void *ptr, void *data) @@ -370,7 +383,7 @@ static int per_file_stats(int id, void *ptr, void *data) #define print_file_stats(m, name, stats) do { \ if (stats.count) \ - seq_printf(m, "%s: %u objects, %zu bytes (%zu active, %zu inactive, %zu global, %zu shared, %zu unbound)\n", \ + seq_printf(m, "%s: %lu objects, %llu bytes (%llu active, %llu inactive, %llu global, %llu shared, %llu unbound)\n", \ name, \ stats.count, \ stats.total, \ @@ -405,7 +418,7 @@ static void print_batch_pool_stats(struct seq_file *m, #define count_vmas(list, member) do { \ list_for_each_entry(vma, list, member) { \ - size += i915_gem_obj_ggtt_size(vma->obj); \ + size += i915_gem_obj_total_ggtt_size(vma->obj); \ ++count; \ if (vma->obj->map_and_fenceable) { \ mappable_size += i915_gem_obj_ggtt_size(vma->obj); \ @@ -420,7 +433,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data) struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 count, mappable_count, purgeable_count; - size_t size, mappable_size, purgeable_size; + u64 size, mappable_size, purgeable_size; struct drm_i915_gem_object *obj; struct i915_address_space *vm = &dev_priv->gtt.base; struct drm_file *file; @@ -437,17 +450,17 @@ static int i915_gem_object_info(struct seq_file *m, void* data) size = count = mappable_size = mappable_count = 0; count_objects(&dev_priv->mm.bound_list, global_list); - seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n", + seq_printf(m, "%u [%u] objects, %llu [%llu] bytes in gtt\n", count, mappable_count, size, mappable_size); size = count = mappable_size = mappable_count = 0; count_vmas(&vm->active_list, mm_list); - seq_printf(m, " %u [%u] active objects, %zu [%zu] bytes\n", + seq_printf(m, " %u [%u] active objects, %llu [%llu] bytes\n", count, mappable_count, size, mappable_size); size = count = mappable_size = mappable_count = 0; count_vmas(&vm->inactive_list, mm_list); - seq_printf(m, " %u [%u] inactive objects, %zu [%zu] bytes\n", + seq_printf(m, " %u [%u] inactive objects, %llu [%llu] bytes\n", count, mappable_count, size, mappable_size); size = count = purgeable_size = purgeable_count = 0; @@ -456,7 +469,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data) if (obj->madv == I915_MADV_DONTNEED) purgeable_size += obj->base.size, ++purgeable_count; } - seq_printf(m, "%u unbound objects, %zu bytes\n", count, size); + seq_printf(m, "%u unbound objects, %llu bytes\n", count, size); size = count = mappable_size = mappable_count = 0; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { @@ -473,16 +486,16 @@ static int i915_gem_object_info(struct seq_file *m, void* data) ++purgeable_count; } } - seq_printf(m, "%u purgeable objects, %zu bytes\n", + seq_printf(m, "%u purgeable objects, %llu bytes\n", purgeable_count, purgeable_size); - seq_printf(m, "%u pinned mappable objects, %zu bytes\n", + seq_printf(m, "%u pinned mappable objects, %llu bytes\n", mappable_count, mappable_size); - seq_printf(m, "%u fault mappable objects, %zu bytes\n", + seq_printf(m, "%u fault mappable objects, %llu bytes\n", count, size); - seq_printf(m, "%zu [%lu] gtt total\n", + seq_printf(m, "%llu [%llu] gtt total\n", dev_priv->gtt.base.total, - dev_priv->gtt.mappable_end - dev_priv->gtt.base.start); + (u64)dev_priv->gtt.mappable_end - dev_priv->gtt.base.start); seq_putc(m, '\n'); print_batch_pool_stats(m, dev_priv); @@ -519,7 +532,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) uintptr_t list = (uintptr_t) node->info_ent->data; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; - size_t total_obj_size, total_gtt_size; + u64 total_obj_size, total_gtt_size; int count, ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -535,13 +548,13 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) describe_obj(m, obj); seq_putc(m, '\n'); total_obj_size += obj->base.size; - total_gtt_size += i915_gem_obj_ggtt_size(obj); + total_gtt_size += i915_gem_obj_total_ggtt_size(obj); count++; } mutex_unlock(&dev->struct_mutex); - seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n", + seq_printf(m, "Total %d objects, %llu bytes, %llu GTT size\n", count, total_obj_size, total_gtt_size); return 0; @@ -943,7 +956,6 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data) if (ret) return ret; - seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start); seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs); for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj; @@ -1132,9 +1144,9 @@ static int i915_frequency_info(struct seq_file *m, void *unused) (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); } else if (IS_GEN6(dev) || (IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) || IS_BROADWELL(dev) || IS_GEN9(dev)) { - u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); - u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS); - u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + u32 rp_state_limits; + u32 gt_perf_status; + u32 rp_state_cap; u32 rpmodectl, rpinclimit, rpdeclimit; u32 rpstat, cagf, reqf; u32 rpupei, rpcurup, rpprevup; @@ -1142,6 +1154,15 @@ static int i915_frequency_info(struct seq_file *m, void *unused) u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; int max_freq; + rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS); + if (IS_BROXTON(dev)) { + rp_state_cap = I915_READ(BXT_RP_STATE_CAP); + gt_perf_status = I915_READ(BXT_GT_PERF_STATUS); + } else { + rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); + } + /* RPSTAT1 is in the GT power well */ ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) @@ -1229,18 +1250,23 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_printf(m, "Down threshold: %d%%\n", dev_priv->rps.down_threshold); - max_freq = (rp_state_cap & 0xff0000) >> 16; - max_freq *= (IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1); + max_freq = (IS_BROXTON(dev) ? rp_state_cap >> 0 : + rp_state_cap >> 16) & 0xff; + max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ? + GEN9_FREQ_SCALER : 1); seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", intel_gpu_freq(dev_priv, max_freq)); max_freq = (rp_state_cap & 0xff00) >> 8; - max_freq *= (IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1); + max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ? + GEN9_FREQ_SCALER : 1); seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", intel_gpu_freq(dev_priv, max_freq)); - max_freq = rp_state_cap & 0xff; - max_freq *= (IS_SKYLAKE(dev) ? GEN9_FREQ_SCALER : 1); + max_freq = (IS_BROXTON(dev) ? rp_state_cap >> 16 : + rp_state_cap >> 0) & 0xff; + max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ? + GEN9_FREQ_SCALER : 1); seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", intel_gpu_freq(dev_priv, max_freq)); seq_printf(m, "Max overclocked frequency: %dMHz\n", @@ -1289,6 +1315,10 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_puts(m, "no P-state info available\n"); } + seq_printf(m, "Current CD clock frequency: %d kHz\n", dev_priv->cdclk_freq); + seq_printf(m, "Max CD clock frequency: %d kHz\n", dev_priv->max_cdclk_freq); + seq_printf(m, "Max pixel clock frequency: %d kHz\n", dev_priv->max_dotclk_freq); + out: intel_runtime_pm_put(dev_priv); return ret; @@ -1362,17 +1392,16 @@ static int ironlake_drpc_info(struct seq_file *m) intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); - seq_printf(m, "HD boost: %s\n", (rgvmodectl & MEMMODE_BOOST_EN) ? - "yes" : "no"); + seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN)); seq_printf(m, "Boost freq: %d\n", (rgvmodectl & MEMMODE_BOOST_FREQ_MASK) >> MEMMODE_BOOST_FREQ_SHIFT); seq_printf(m, "HW control enabled: %s\n", - rgvmodectl & MEMMODE_HWIDLE_EN ? "yes" : "no"); + yesno(rgvmodectl & MEMMODE_HWIDLE_EN)); seq_printf(m, "SW control enabled: %s\n", - rgvmodectl & MEMMODE_SWMODE_EN ? "yes" : "no"); + yesno(rgvmodectl & MEMMODE_SWMODE_EN)); seq_printf(m, "Gated voltage change: %s\n", - rgvmodectl & MEMMODE_RCLK_GATE ? "yes" : "no"); + yesno(rgvmodectl & MEMMODE_RCLK_GATE)); seq_printf(m, "Starting frequency: P%d\n", (rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT); seq_printf(m, "Max P-state: P%d\n", @@ -1381,7 +1410,7 @@ static int ironlake_drpc_info(struct seq_file *m) seq_printf(m, "RS1 VID: %d\n", (crstandvid & 0x3f)); seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f)); seq_printf(m, "Render standby enabled: %s\n", - (rstdbyctl & RCX_SW_EXIT) ? "no" : "yes"); + yesno(!(rstdbyctl & RCX_SW_EXIT))); seq_puts(m, "Current RS state: "); switch (rstdbyctl & RSX_STATUS_MASK) { case RSX_STATUS_ON: @@ -1497,7 +1526,7 @@ static int gen6_drpc_info(struct seq_file *m) seq_printf(m, "RC information accurate: %s\n", yesno(count < 51)); } - gt_core_status = readl(dev_priv->regs + GEN6_GT_CORE_STATUS); + gt_core_status = I915_READ_FW(GEN6_GT_CORE_STATUS); trace_i915_reg_rw(false, GEN6_GT_CORE_STATUS, gt_core_status, 4, true); rpmodectl1 = I915_READ(GEN6_RP_CONTROL); @@ -1581,6 +1610,21 @@ static int i915_drpc_info(struct seq_file *m, void *unused) return ironlake_drpc_info(m); } +static int i915_frontbuffer_tracking(struct seq_file *m, void *unused) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + seq_printf(m, "FB tracking busy bits: 0x%08x\n", + dev_priv->fb_tracking.busy_bits); + + seq_printf(m, "FB tracking flip bits: 0x%08x\n", + dev_priv->fb_tracking.flip_bits); + + return 0; +} + static int i915_fbc_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; @@ -1593,51 +1637,20 @@ static int i915_fbc_status(struct seq_file *m, void *unused) } intel_runtime_pm_get(dev_priv); + mutex_lock(&dev_priv->fbc.lock); - if (intel_fbc_enabled(dev)) { + if (intel_fbc_is_active(dev_priv)) seq_puts(m, "FBC enabled\n"); - } else { - seq_puts(m, "FBC disabled: "); - switch (dev_priv->fbc.no_fbc_reason) { - case FBC_OK: - seq_puts(m, "FBC actived, but currently disabled in hardware"); - break; - case FBC_UNSUPPORTED: - seq_puts(m, "unsupported by this chipset"); - break; - case FBC_NO_OUTPUT: - seq_puts(m, "no outputs"); - break; - case FBC_STOLEN_TOO_SMALL: - seq_puts(m, "not enough stolen memory"); - break; - case FBC_UNSUPPORTED_MODE: - seq_puts(m, "mode not supported"); - break; - case FBC_MODE_TOO_LARGE: - seq_puts(m, "mode too large"); - break; - case FBC_BAD_PLANE: - seq_puts(m, "FBC unsupported on plane"); - break; - case FBC_NOT_TILED: - seq_puts(m, "scanout buffer not tiled"); - break; - case FBC_MULTIPLE_PIPES: - seq_puts(m, "multiple pipes are enabled"); - break; - case FBC_MODULE_PARAM: - seq_puts(m, "disabled per module param (default off)"); - break; - case FBC_CHIP_DEFAULT: - seq_puts(m, "disabled per chip default"); - break; - default: - seq_puts(m, "unknown reason"); - } - seq_putc(m, '\n'); - } + else + seq_printf(m, "FBC disabled: %s\n", + dev_priv->fbc.no_fbc_reason); + if (INTEL_INFO(dev_priv)->gen >= 7) + seq_printf(m, "Compressing: %s\n", + yesno(I915_READ(FBC_STATUS2) & + FBC_COMPRESSION_MASK)); + + mutex_unlock(&dev_priv->fbc.lock); intel_runtime_pm_put(dev_priv); return 0; @@ -1651,9 +1664,7 @@ static int i915_fbc_fc_get(void *data, u64 *val) if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev)) return -ENODEV; - drm_modeset_lock_all(dev); *val = dev_priv->fbc.false_color; - drm_modeset_unlock_all(dev); return 0; } @@ -1667,7 +1678,7 @@ static int i915_fbc_fc_set(void *data, u64 val) if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev)) return -ENODEV; - drm_modeset_lock_all(dev); + mutex_lock(&dev_priv->fbc.lock); reg = I915_READ(ILK_DPFC_CONTROL); dev_priv->fbc.false_color = val; @@ -1676,7 +1687,7 @@ static int i915_fbc_fc_set(void *data, u64 val) (reg | FBC_CTL_FALSE_COLOR) : (reg & ~FBC_CTL_FALSE_COLOR)); - drm_modeset_unlock_all(dev); + mutex_unlock(&dev_priv->fbc.lock); return 0; } @@ -1778,8 +1789,9 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) struct drm_i915_private *dev_priv = dev->dev_private; int ret = 0; int gpu_freq, ia_freq; + unsigned int max_gpu_freq, min_gpu_freq; - if (!(IS_GEN6(dev) || IS_GEN7(dev))) { + if (!HAS_CORE_RING_FREQ(dev)) { seq_puts(m, "unsupported on this chipset\n"); return 0; } @@ -1792,17 +1804,28 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) if (ret) goto out; + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + /* Convert GT frequency to 50 HZ units */ + min_gpu_freq = + dev_priv->rps.min_freq_softlimit / GEN9_FREQ_SCALER; + max_gpu_freq = + dev_priv->rps.max_freq_softlimit / GEN9_FREQ_SCALER; + } else { + min_gpu_freq = dev_priv->rps.min_freq_softlimit; + max_gpu_freq = dev_priv->rps.max_freq_softlimit; + } + seq_puts(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\tEffective Ring freq (MHz)\n"); - for (gpu_freq = dev_priv->rps.min_freq_softlimit; - gpu_freq <= dev_priv->rps.max_freq_softlimit; - gpu_freq++) { + for (gpu_freq = min_gpu_freq; gpu_freq <= max_gpu_freq; gpu_freq++) { ia_freq = gpu_freq; sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_MIN_FREQ_TABLE, &ia_freq); seq_printf(m, "%d\t\t%d\t\t\t\t%d\n", - intel_gpu_freq(dev_priv, gpu_freq), + intel_gpu_freq(dev_priv, (gpu_freq * + (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ? + GEN9_FREQ_SCALER : 1))), ((ia_freq >> 0) & 0xff) * 100, ((ia_freq >> 8) & 0xff) * 100); } @@ -1831,7 +1854,7 @@ static int i915_opregion(struct seq_file *m, void *unused) goto out; if (opregion->header) { - memcpy_fromio(data, opregion->header, OPREGION_SIZE); + memcpy(data, opregion->header, OPREGION_SIZE); seq_write(m, data, OPREGION_SIZE); } @@ -1846,29 +1869,29 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct intel_fbdev *ifbdev = NULL; - struct intel_framebuffer *fb; - -#ifdef CONFIG_DRM_I915_FBDEV - struct drm_i915_private *dev_priv = dev->dev_private; - - ifbdev = dev_priv->fbdev; - fb = to_intel_framebuffer(ifbdev->helper.fb); - - seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", - fb->base.width, - fb->base.height, - fb->base.depth, - fb->base.bits_per_pixel, - fb->base.modifier[0], - atomic_read(&fb->base.refcount.refcount)); - describe_obj(m, fb->obj); - seq_putc(m, '\n'); + struct intel_framebuffer *fbdev_fb = NULL; + struct drm_framebuffer *drm_fb; + +#ifdef CONFIG_DRM_FBDEV_EMULATION + if (to_i915(dev)->fbdev) { + fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb); + + seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", + fbdev_fb->base.width, + fbdev_fb->base.height, + fbdev_fb->base.depth, + fbdev_fb->base.bits_per_pixel, + fbdev_fb->base.modifier[0], + atomic_read(&fbdev_fb->base.refcount.refcount)); + describe_obj(m, fbdev_fb->obj); + seq_putc(m, '\n'); + } #endif mutex_lock(&dev->mode_config.fb_lock); - list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) { - if (ifbdev && &fb->base == ifbdev->helper.fb) + drm_for_each_fb(drm_fb, dev) { + struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); + if (fb == fbdev_fb) continue; seq_printf(m, "user size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", @@ -1975,7 +1998,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, return; } - page = i915_gem_object_get_page(ctx_obj, 1); + page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); if (!WARN_ON(page == NULL)) { reg_state = kmap_atomic(page); @@ -2055,8 +2078,8 @@ static int i915_execlists(struct seq_file *m, void *data) seq_printf(m, "%s\n", ring->name); - status = I915_READ(RING_EXECLIST_STATUS(ring)); - ctx_id = I915_READ(RING_EXECLIST_STATUS(ring) + 4); + status = I915_READ(RING_EXECLIST_STATUS_LO(ring)); + ctx_id = I915_READ(RING_EXECLIST_STATUS_HI(ring)); seq_printf(m, "\tExeclist status: 0x%08X, context: %u\n", status, ctx_id); @@ -2071,8 +2094,8 @@ static int i915_execlists(struct seq_file *m, void *data) read_pointer, write_pointer); for (i = 0; i < 6; i++) { - status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 8*i); - ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 8*i + 4); + status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, i)); + ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, i)); seq_printf(m, "\tStatus buffer %d: 0x%08X, context: %u\n", i, status, ctx_id); @@ -2217,10 +2240,9 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) for_each_ring(ring, dev_priv, unused) { seq_printf(m, "%s\n", ring->name); for (i = 0; i < 4; i++) { - u32 offset = 0x270 + i * 8; - u64 pdp = I915_READ(ring->mmio_base + offset + 4); + u64 pdp = I915_READ(GEN8_RING_PDP_UDW(ring, i)); pdp <<= 32; - pdp |= I915_READ(ring->mmio_base + offset); + pdp |= I915_READ(GEN8_RING_PDP_LDW(ring, i)); seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp); } } @@ -2230,7 +2252,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring; - struct drm_file *file; int i; if (INTEL_INFO(dev)->gen == 6) @@ -2248,18 +2269,11 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; seq_puts(m, "aliasing PPGTT:\n"); - seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd.pd_offset); + seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd.base.ggtt_offset); ppgtt->debug_dump(ppgtt, m); } - list_for_each_entry_reverse(file, &dev->filelist, lhead) { - struct drm_i915_file_private *file_priv = file->driver_priv; - - seq_printf(m, "proc: %s\n", - get_pid_task(file->pid, PIDTYPE_PID)->comm); - idr_for_each(&file_priv->context_idr, per_file_ctx, m); - } seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK)); } @@ -2268,6 +2282,7 @@ static int i915_ppgtt_info(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_file *file; int ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) @@ -2279,10 +2294,26 @@ static int i915_ppgtt_info(struct seq_file *m, void *data) else if (INTEL_INFO(dev)->gen >= 6) gen6_ppgtt_info(m, dev); + list_for_each_entry_reverse(file, &dev->filelist, lhead) { + struct drm_i915_file_private *file_priv = file->driver_priv; + struct task_struct *task; + + task = get_pid_task(file->pid, PIDTYPE_PID); + if (!task) { + ret = -ESRCH; + goto out_put; + } + seq_printf(m, "\nproc: %s\n", task->comm); + put_task_struct(task); + idr_for_each(&file_priv->context_idr, per_file_ctx, + (void *)(unsigned long)m); + } + +out_put: intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); - return 0; + return ret; } static int count_irq_waiters(struct drm_i915_private *i915) @@ -2352,6 +2383,153 @@ static int i915_llc(struct seq_file *m, void *data) return 0; } +static int i915_guc_load_status_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct drm_i915_private *dev_priv = node->minor->dev->dev_private; + struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; + u32 tmp, i; + + if (!HAS_GUC_UCODE(dev_priv->dev)) + return 0; + + seq_printf(m, "GuC firmware status:\n"); + seq_printf(m, "\tpath: %s\n", + guc_fw->guc_fw_path); + seq_printf(m, "\tfetch: %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status)); + seq_printf(m, "\tload: %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); + seq_printf(m, "\tversion wanted: %d.%d\n", + guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted); + seq_printf(m, "\tversion found: %d.%d\n", + guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found); + seq_printf(m, "\theader: offset is %d; size = %d\n", + guc_fw->header_offset, guc_fw->header_size); + seq_printf(m, "\tuCode: offset is %d; size = %d\n", + guc_fw->ucode_offset, guc_fw->ucode_size); + seq_printf(m, "\tRSA: offset is %d; size = %d\n", + guc_fw->rsa_offset, guc_fw->rsa_size); + + tmp = I915_READ(GUC_STATUS); + + seq_printf(m, "\nGuC status 0x%08x:\n", tmp); + seq_printf(m, "\tBootrom status = 0x%x\n", + (tmp & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); + seq_printf(m, "\tuKernel status = 0x%x\n", + (tmp & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); + seq_printf(m, "\tMIA Core status = 0x%x\n", + (tmp & GS_MIA_MASK) >> GS_MIA_SHIFT); + seq_puts(m, "\nScratch registers:\n"); + for (i = 0; i < 16; i++) + seq_printf(m, "\t%2d: \t0x%x\n", i, I915_READ(SOFT_SCRATCH(i))); + + return 0; +} + +static void i915_guc_client_info(struct seq_file *m, + struct drm_i915_private *dev_priv, + struct i915_guc_client *client) +{ + struct intel_engine_cs *ring; + uint64_t tot = 0; + uint32_t i; + + seq_printf(m, "\tPriority %d, GuC ctx index: %u, PD offset 0x%x\n", + client->priority, client->ctx_index, client->proc_desc_offset); + seq_printf(m, "\tDoorbell id %d, offset: 0x%x, cookie 0x%x\n", + client->doorbell_id, client->doorbell_offset, client->cookie); + seq_printf(m, "\tWQ size %d, offset: 0x%x, tail %d\n", + client->wq_size, client->wq_offset, client->wq_tail); + + seq_printf(m, "\tFailed to queue: %u\n", client->q_fail); + seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail); + seq_printf(m, "\tLast submission result: %d\n", client->retcode); + + for_each_ring(ring, dev_priv, i) { + seq_printf(m, "\tSubmissions: %llu %s\n", + client->submissions[i], + ring->name); + tot += client->submissions[i]; + } + seq_printf(m, "\tTotal: %llu\n", tot); +} + +static int i915_guc_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc guc; + struct i915_guc_client client = {}; + struct intel_engine_cs *ring; + enum intel_ring_id i; + u64 total = 0; + + if (!HAS_GUC_SCHED(dev_priv->dev)) + return 0; + + if (mutex_lock_interruptible(&dev->struct_mutex)) + return 0; + + /* Take a local copy of the GuC data, so we can dump it at leisure */ + guc = dev_priv->guc; + if (guc.execbuf_client) + client = *guc.execbuf_client; + + mutex_unlock(&dev->struct_mutex); + + seq_printf(m, "GuC total action count: %llu\n", guc.action_count); + seq_printf(m, "GuC action failure count: %u\n", guc.action_fail); + seq_printf(m, "GuC last action command: 0x%x\n", guc.action_cmd); + seq_printf(m, "GuC last action status: 0x%x\n", guc.action_status); + seq_printf(m, "GuC last action error code: %d\n", guc.action_err); + + seq_printf(m, "\nGuC submissions:\n"); + for_each_ring(ring, dev_priv, i) { + seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x %9d\n", + ring->name, guc.submissions[i], + guc.last_seqno[i], guc.last_seqno[i]); + total += guc.submissions[i]; + } + seq_printf(m, "\t%s: %llu\n", "Total", total); + + seq_printf(m, "\nGuC execbuf client @ %p:\n", guc.execbuf_client); + i915_guc_client_info(m, dev_priv, &client); + + /* Add more as required ... */ + + return 0; +} + +static int i915_guc_log_dump(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_object *log_obj = dev_priv->guc.log_obj; + u32 *log; + int i = 0, pg; + + if (!log_obj) + return 0; + + for (pg = 0; pg < log_obj->base.size / PAGE_SIZE; pg++) { + log = kmap_atomic(i915_gem_object_get_page(log_obj, pg)); + + for (i = 0; i < PAGE_SIZE / sizeof(u32); i += 4) + seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n", + *(log + i), *(log + i + 1), + *(log + i + 2), *(log + i + 3)); + + kunmap_atomic(log); + } + + seq_putc(m, '\n'); + + return 0; +} + static int i915_edp_psr_status(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; @@ -2380,7 +2558,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) yesno(work_busy(&dev_priv->psr.work.work))); if (HAS_DDI(dev)) - enabled = I915_READ(EDP_PSR_CTL(dev)) & EDP_PSR_ENABLE; + enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE; else { for_each_pipe(dev_priv, pipe) { stat[pipe] = I915_READ(VLV_PSRSTAT(pipe)) & @@ -2400,9 +2578,12 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) } seq_puts(m, "\n"); - /* CHV PSR has no kind of performance counter */ - if (HAS_DDI(dev)) { - psrperf = I915_READ(EDP_PSR_PERF_CNT(dev)) & + /* + * VLV/CHV PSR has no kind of performance counter + * SKL+ Perf counter is reset to 0 everytime DC state is entered + */ + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + psrperf = I915_READ(EDP_PSR_PERF_CNT) & EDP_PSR_PERF_CNT_MASK; seq_printf(m, "Performance_Counter: %u\n", psrperf); @@ -2479,13 +2660,13 @@ static int i915_energy_uJ(struct seq_file *m, void *data) return 0; } -static int i915_pc8_status(struct seq_file *m, void *unused) +static int i915_runtime_pm_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) { + if (!HAS_RUNTIME_PM(dev)) { seq_puts(m, "not supported\n"); return 0; } @@ -2493,77 +2674,16 @@ static int i915_pc8_status(struct seq_file *m, void *unused) seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->mm.busy)); seq_printf(m, "IRQs disabled: %s\n", yesno(!intel_irqs_enabled(dev_priv))); +#ifdef CONFIG_PM + seq_printf(m, "Usage count: %d\n", + atomic_read(&dev->dev->power.usage_count)); +#else + seq_printf(m, "Device Power Management (CONFIG_PM) disabled\n"); +#endif return 0; } -static const char *power_domain_str(enum intel_display_power_domain domain) -{ - switch (domain) { - case POWER_DOMAIN_PIPE_A: - return "PIPE_A"; - case POWER_DOMAIN_PIPE_B: - return "PIPE_B"; - case POWER_DOMAIN_PIPE_C: - return "PIPE_C"; - case POWER_DOMAIN_PIPE_A_PANEL_FITTER: - return "PIPE_A_PANEL_FITTER"; - case POWER_DOMAIN_PIPE_B_PANEL_FITTER: - return "PIPE_B_PANEL_FITTER"; - case POWER_DOMAIN_PIPE_C_PANEL_FITTER: - return "PIPE_C_PANEL_FITTER"; - case POWER_DOMAIN_TRANSCODER_A: - return "TRANSCODER_A"; - case POWER_DOMAIN_TRANSCODER_B: - return "TRANSCODER_B"; - case POWER_DOMAIN_TRANSCODER_C: - return "TRANSCODER_C"; - case POWER_DOMAIN_TRANSCODER_EDP: - return "TRANSCODER_EDP"; - case POWER_DOMAIN_PORT_DDI_A_2_LANES: - return "PORT_DDI_A_2_LANES"; - case POWER_DOMAIN_PORT_DDI_A_4_LANES: - return "PORT_DDI_A_4_LANES"; - case POWER_DOMAIN_PORT_DDI_B_2_LANES: - return "PORT_DDI_B_2_LANES"; - case POWER_DOMAIN_PORT_DDI_B_4_LANES: - return "PORT_DDI_B_4_LANES"; - case POWER_DOMAIN_PORT_DDI_C_2_LANES: - return "PORT_DDI_C_2_LANES"; - case POWER_DOMAIN_PORT_DDI_C_4_LANES: - return "PORT_DDI_C_4_LANES"; - case POWER_DOMAIN_PORT_DDI_D_2_LANES: - return "PORT_DDI_D_2_LANES"; - case POWER_DOMAIN_PORT_DDI_D_4_LANES: - return "PORT_DDI_D_4_LANES"; - case POWER_DOMAIN_PORT_DSI: - return "PORT_DSI"; - case POWER_DOMAIN_PORT_CRT: - return "PORT_CRT"; - case POWER_DOMAIN_PORT_OTHER: - return "PORT_OTHER"; - case POWER_DOMAIN_VGA: - return "VGA"; - case POWER_DOMAIN_AUDIO: - return "AUDIO"; - case POWER_DOMAIN_PLLS: - return "PLLS"; - case POWER_DOMAIN_AUX_A: - return "AUX_A"; - case POWER_DOMAIN_AUX_B: - return "AUX_B"; - case POWER_DOMAIN_AUX_C: - return "AUX_C"; - case POWER_DOMAIN_AUX_D: - return "AUX_D"; - case POWER_DOMAIN_INIT: - return "INIT"; - default: - MISSING_CASE(domain); - return "?"; - } -} - static int i915_power_domain_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; @@ -2589,7 +2709,7 @@ static int i915_power_domain_info(struct seq_file *m, void *unused) continue; seq_printf(m, " %-23s %d\n", - power_domain_str(power_domain), + intel_display_power_domain_str(power_domain), power_domains->domain_use_count[power_domain]); } } @@ -2599,6 +2719,51 @@ static int i915_power_domain_info(struct seq_file *m, void *unused) return 0; } +static int i915_dmc_info(struct seq_file *m, void *unused) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_csr *csr; + + if (!HAS_CSR(dev)) { + seq_puts(m, "not supported\n"); + return 0; + } + + csr = &dev_priv->csr; + + intel_runtime_pm_get(dev_priv); + + seq_printf(m, "fw loaded: %s\n", yesno(csr->dmc_payload != NULL)); + seq_printf(m, "path: %s\n", csr->fw_path); + + if (!csr->dmc_payload) + goto out; + + seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version), + CSR_VERSION_MINOR(csr->version)); + + if (IS_SKYLAKE(dev) && csr->version >= CSR_VERSION(1, 6)) { + seq_printf(m, "DC3 -> DC5 count: %d\n", + I915_READ(SKL_CSR_DC3_DC5_COUNT)); + seq_printf(m, "DC5 -> DC6 count: %d\n", + I915_READ(SKL_CSR_DC5_DC6_COUNT)); + } else if (IS_BROXTON(dev) && csr->version >= CSR_VERSION(1, 4)) { + seq_printf(m, "DC3 -> DC5 count: %d\n", + I915_READ(BXT_CSR_DC3_DC5_COUNT)); + } + +out: + seq_printf(m, "program base: 0x%08x\n", I915_READ(CSR_PROGRAM(0))); + seq_printf(m, "ssp base: 0x%08x\n", I915_READ(CSR_SSP_BASE)); + seq_printf(m, "htp: 0x%08x\n", I915_READ(CSR_HTP_SKL)); + + intel_runtime_pm_put(dev_priv); + + return 0; +} + static void intel_seq_print_mode(struct seq_file *m, int tabs, struct drm_display_mode *mode) { @@ -2652,11 +2817,13 @@ static void intel_crtc_info(struct seq_file *m, struct intel_crtc *intel_crtc) struct drm_device *dev = node->minor->dev; struct drm_crtc *crtc = &intel_crtc->base; struct intel_encoder *intel_encoder; + struct drm_plane_state *plane_state = crtc->primary->state; + struct drm_framebuffer *fb = plane_state->fb; - if (crtc->primary->fb) + if (fb) seq_printf(m, "\tfb: %d, pos: %dx%d, size: %dx%d\n", - crtc->primary->fb->base.id, crtc->x, crtc->y, - crtc->primary->fb->width, crtc->primary->fb->height); + fb->base.id, plane_state->src_x >> 16, + plane_state->src_y >> 16, fb->width, fb->height); else seq_puts(m, "\tprimary plane disabled\n"); for_each_encoder_on_crtc(dev, crtc, intel_encoder) @@ -2678,8 +2845,7 @@ static void intel_dp_info(struct seq_file *m, struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]); - seq_printf(m, "\taudio support: %s\n", intel_dp->has_audio ? "yes" : - "no"); + seq_printf(m, "\taudio support: %s\n", yesno(intel_dp->has_audio)); if (intel_encoder->type == INTEL_OUTPUT_EDP) intel_panel_info(m, &intel_connector->panel); } @@ -2690,8 +2856,7 @@ static void intel_hdmi_info(struct seq_file *m, struct intel_encoder *intel_encoder = intel_connector->encoder; struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); - seq_printf(m, "\taudio support: %s\n", intel_hdmi->has_audio ? "yes" : - "no"); + seq_printf(m, "\taudio support: %s\n", yesno(intel_hdmi->has_audio)); } static void intel_lvds_info(struct seq_file *m, @@ -2741,7 +2906,7 @@ static bool cursor_active(struct drm_device *dev, int pipe) u32 state; if (IS_845G(dev) || IS_I865G(dev)) - state = I915_READ(_CURACNTR) & CURSOR_ENABLE; + state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE; else state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE; @@ -2766,6 +2931,107 @@ static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y) return cursor_active(dev, pipe); } +static const char *plane_type(enum drm_plane_type type) +{ + switch (type) { + case DRM_PLANE_TYPE_OVERLAY: + return "OVL"; + case DRM_PLANE_TYPE_PRIMARY: + return "PRI"; + case DRM_PLANE_TYPE_CURSOR: + return "CUR"; + /* + * Deliberately omitting default: to generate compiler warnings + * when a new drm_plane_type gets added. + */ + } + + return "unknown"; +} + +static const char *plane_rotation(unsigned int rotation) +{ + static char buf[48]; + /* + * According to doc only one DRM_ROTATE_ is allowed but this + * will print them all to visualize if the values are misused + */ + snprintf(buf, sizeof(buf), + "%s%s%s%s%s%s(0x%08x)", + (rotation & BIT(DRM_ROTATE_0)) ? "0 " : "", + (rotation & BIT(DRM_ROTATE_90)) ? "90 " : "", + (rotation & BIT(DRM_ROTATE_180)) ? "180 " : "", + (rotation & BIT(DRM_ROTATE_270)) ? "270 " : "", + (rotation & BIT(DRM_REFLECT_X)) ? "FLIPX " : "", + (rotation & BIT(DRM_REFLECT_Y)) ? "FLIPY " : "", + rotation); + + return buf; +} + +static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct intel_plane *intel_plane; + + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + struct drm_plane_state *state; + struct drm_plane *plane = &intel_plane->base; + + if (!plane->state) { + seq_puts(m, "plane->state is NULL!\n"); + continue; + } + + state = plane->state; + + seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n", + plane->base.id, + plane_type(intel_plane->base.type), + state->crtc_x, state->crtc_y, + state->crtc_w, state->crtc_h, + (state->src_x >> 16), + ((state->src_x & 0xffff) * 15625) >> 10, + (state->src_y >> 16), + ((state->src_y & 0xffff) * 15625) >> 10, + (state->src_w >> 16), + ((state->src_w & 0xffff) * 15625) >> 10, + (state->src_h >> 16), + ((state->src_h & 0xffff) * 15625) >> 10, + state->fb ? drm_get_format_name(state->fb->pixel_format) : "N/A", + plane_rotation(state->rotation)); + } +} + +static void intel_scaler_info(struct seq_file *m, struct intel_crtc *intel_crtc) +{ + struct intel_crtc_state *pipe_config; + int num_scalers = intel_crtc->num_scalers; + int i; + + pipe_config = to_intel_crtc_state(intel_crtc->base.state); + + /* Not all platformas have a scaler */ + if (num_scalers) { + seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d", + num_scalers, + pipe_config->scaler_state.scaler_users, + pipe_config->scaler_state.scaler_id); + + for (i = 0; i < SKL_NUM_SCALERS; i++) { + struct intel_scaler *sc = + &pipe_config->scaler_state.scalers[i]; + + seq_printf(m, ", scalers[%d]: use=%s, mode=%x", + i, yesno(sc->in_use), sc->mode); + } + seq_puts(m, "\n"); + } else { + seq_puts(m, "\tNo scalers available on this platform\n"); + } +} + static int i915_display_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; @@ -2780,13 +3046,18 @@ static int i915_display_info(struct seq_file *m, void *unused) seq_printf(m, "---------\n"); for_each_intel_crtc(dev, crtc) { bool active; + struct intel_crtc_state *pipe_config; int x, y; - seq_printf(m, "CRTC %d: pipe: %c, active=%s (size=%dx%d)\n", + pipe_config = to_intel_crtc_state(crtc->base.state); + + seq_printf(m, "CRTC %d: pipe: %c, active=%s, (size=%dx%d), dither=%s, bpp=%d\n", crtc->base.base.id, pipe_name(crtc->pipe), - yesno(crtc->active), crtc->config->pipe_src_w, - crtc->config->pipe_src_h); - if (crtc->active) { + yesno(pipe_config->base.active), + pipe_config->pipe_src_w, pipe_config->pipe_src_h, + yesno(pipe_config->dither), pipe_config->pipe_bpp); + + if (pipe_config->base.active) { intel_crtc_info(m, crtc); active = cursor_position(dev, crtc->pipe, &x, &y); @@ -2795,6 +3066,8 @@ static int i915_display_info(struct seq_file *m, void *unused) x, y, crtc->base.cursor->state->crtc_w, crtc->base.cursor->state->crtc_h, crtc->cursor_addr, yesno(active)); + intel_scaler_info(m, crtc); + intel_plane_info(m, crtc); } seq_printf(m, "\tunderrun reporting: cpu=%s pch=%s \n", @@ -2929,7 +3202,8 @@ static int i915_wa_registers(struct seq_file *m, void *unused) seq_printf(m, "Workarounds applied: %d\n", dev_priv->workarounds.count); for (i = 0; i < dev_priv->workarounds.count; ++i) { - u32 addr, mask, value, read; + i915_reg_t addr; + u32 mask, value, read; bool ok; addr = dev_priv->workarounds.reg[i].addr; @@ -2938,7 +3212,7 @@ static int i915_wa_registers(struct seq_file *m, void *unused) read = I915_READ(addr); ok = (value & mask) == (read & mask); seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X, read: 0x%08x, status: %s\n", - addr, value, mask, read, ok ? "OK" : "FAIL"); + i915_mmio_reg_offset(addr), value, mask, read, ok ? "OK" : "FAIL"); } intel_runtime_pm_put(dev_priv); @@ -2976,7 +3250,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused) skl_ddb_entry_size(entry)); } - entry = &ddb->cursor[pipe]; + entry = &ddb->plane[pipe][PLANE_CURSOR]; seq_printf(m, " %-13s%8u%8u%8u\n", "Cursor", entry->start, entry->end, skl_ddb_entry_size(entry)); } @@ -3027,7 +3301,7 @@ static void drrs_status_per_crtc(struct seq_file *m, seq_puts(m, "\n\n"); - if (intel_crtc->config->has_drrs) { + if (to_intel_crtc_state(intel_crtc->base.state)->has_drrs) { struct intel_panel *panel; mutex_lock(&drrs->mutex); @@ -3079,7 +3353,7 @@ static int i915_drrs_status(struct seq_file *m, void *unused) for_each_intel_crtc(dev, intel_crtc) { drm_modeset_lock(&intel_crtc->base.mutex, NULL); - if (intel_crtc->active) { + if (intel_crtc->base.state->active) { active_crtc_cnt++; seq_printf(m, "\nCRTC %d: ", active_crtc_cnt); @@ -3616,53 +3890,40 @@ static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source, return 0; } -static void hsw_trans_edp_pipe_A_crc_wa(struct drm_device *dev) +static void hsw_trans_edp_pipe_A_crc_wa(struct drm_device *dev, bool enable) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]); + struct intel_crtc_state *pipe_config; + struct drm_atomic_state *state; + int ret = 0; drm_modeset_lock_all(dev); - /* - * If we use the eDP transcoder we need to make sure that we don't - * bypass the pfit, since otherwise the pipe CRC source won't work. Only - * relevant on hsw with pipe A when using the always-on power well - * routing. - */ - if (crtc->config->cpu_transcoder == TRANSCODER_EDP && - !crtc->config->pch_pfit.enabled) { - crtc->config->pch_pfit.force_thru = true; - - intel_display_power_get(dev_priv, - POWER_DOMAIN_PIPE_PANEL_FITTER(PIPE_A)); - - intel_crtc_reset(crtc); + state = drm_atomic_state_alloc(dev); + if (!state) { + ret = -ENOMEM; + goto out; } - drm_modeset_unlock_all(dev); -} -static void hsw_undo_trans_edp_pipe_A_crc_wa(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]); - - drm_modeset_lock_all(dev); - /* - * If we use the eDP transcoder we need to make sure that we don't - * bypass the pfit, since otherwise the pipe CRC source won't work. Only - * relevant on hsw with pipe A when using the always-on power well - * routing. - */ - if (crtc->config->pch_pfit.force_thru) { - crtc->config->pch_pfit.force_thru = false; + state->acquire_ctx = drm_modeset_legacy_acquire_ctx(&crtc->base); + pipe_config = intel_atomic_get_crtc_state(state, crtc); + if (IS_ERR(pipe_config)) { + ret = PTR_ERR(pipe_config); + goto out; + } - intel_crtc_reset(crtc); + pipe_config->pch_pfit.force_thru = enable; + if (pipe_config->cpu_transcoder == TRANSCODER_EDP && + pipe_config->pch_pfit.enabled != enable) + pipe_config->base.connectors_changed = true; - intel_display_power_put(dev_priv, - POWER_DOMAIN_PIPE_PANEL_FITTER(PIPE_A)); - } + ret = drm_atomic_commit(state); +out: drm_modeset_unlock_all(dev); + WARN(ret, "Toggling workaround to %i returns %i\n", enable, ret); + if (ret) + drm_atomic_state_free(state); } static int ivb_pipe_crc_ctl_reg(struct drm_device *dev, @@ -3682,7 +3943,7 @@ static int ivb_pipe_crc_ctl_reg(struct drm_device *dev, break; case INTEL_PIPE_CRC_SOURCE_PF: if (IS_HASWELL(dev) && pipe == PIPE_A) - hsw_trans_edp_pipe_A_crc_wa(dev); + hsw_trans_edp_pipe_A_crc_wa(dev, true); *val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PF_IVB; break; @@ -3776,7 +4037,7 @@ static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe, pipe_name(pipe)); drm_modeset_lock(&crtc->base.mutex, NULL); - if (crtc->active) + if (crtc->base.state->active) intel_wait_for_vblank(dev, pipe); drm_modeset_unlock(&crtc->base.mutex); @@ -3794,7 +4055,7 @@ static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe, else if (IS_VALLEYVIEW(dev)) vlv_undo_pipe_scramble_reset(dev, pipe); else if (IS_HASWELL(dev) && pipe == PIPE_A) - hsw_undo_trans_edp_pipe_A_crc_wa(dev); + hsw_trans_edp_pipe_A_crc_wa(dev, false); hsw_enable_ips(crtc); } @@ -3980,24 +4241,14 @@ static ssize_t i915_displayport_test_active_write(struct file *file, { char *input_buffer; int status = 0; - struct seq_file *m; struct drm_device *dev; struct drm_connector *connector; struct list_head *connector_list; struct intel_dp *intel_dp; int val = 0; - m = file->private_data; - if (!m) { - status = -ENODEV; - return status; - } - dev = m->private; + dev = ((struct seq_file *)file->private_data)->private; - if (!dev) { - status = -ENODEV; - return status; - } connector_list = &dev->mode_config.connector_list; if (len == 0) @@ -4021,9 +4272,7 @@ static ssize_t i915_displayport_test_active_write(struct file *file, DRM_MODE_CONNECTOR_DisplayPort) continue; - if (connector->connector_type == - DRM_MODE_CONNECTOR_DisplayPort && - connector->status == connector_status_connected && + if (connector->status == connector_status_connected && connector->encoder != NULL) { intel_dp = enc_to_intel_dp(connector->encoder); status = kstrtoint(input_buffer, 10, &val); @@ -4055,9 +4304,6 @@ static int i915_displayport_test_active_show(struct seq_file *m, void *data) struct list_head *connector_list = &dev->mode_config.connector_list; struct intel_dp *intel_dp; - if (!dev) - return -ENODEV; - list_for_each_entry(connector, connector_list, head) { if (connector->connector_type != @@ -4102,9 +4348,6 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data) struct list_head *connector_list = &dev->mode_config.connector_list; struct intel_dp *intel_dp; - if (!dev) - return -ENODEV; - list_for_each_entry(connector, connector_list, head) { if (connector->connector_type != @@ -4144,9 +4387,6 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data) struct list_head *connector_list = &dev->mode_config.connector_list; struct intel_dp *intel_dp; - if (!dev) - return -ENODEV; - list_for_each_entry(connector, connector_list, head) { if (connector->connector_type != @@ -4183,8 +4423,15 @@ static const struct file_operations i915_displayport_test_type_fops = { static void wm_latency_show(struct seq_file *m, const uint16_t wm[8]) { struct drm_device *dev = m->private; - int num_levels = ilk_wm_max_level(dev) + 1; int level; + int num_levels; + + if (IS_CHERRYVIEW(dev)) + num_levels = 3; + else if (IS_VALLEYVIEW(dev)) + num_levels = 1; + else + num_levels = ilk_wm_max_level(dev) + 1; drm_modeset_lock_all(dev); @@ -4193,9 +4440,9 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8]) /* * - WM1+ latency values in 0.5us units - * - latencies are in us on gen9 + * - latencies are in us on gen9/vlv/chv */ - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_INFO(dev)->gen >= 9 || IS_VALLEYVIEW(dev)) latency *= 10; else if (level > 0) latency *= 5; @@ -4259,7 +4506,7 @@ static int pri_wm_latency_open(struct inode *inode, struct file *file) { struct drm_device *dev = inode->i_private; - if (HAS_GMCH_DISPLAY(dev)) + if (INTEL_INFO(dev)->gen < 5) return -ENODEV; return single_open(file, pri_wm_latency_show, dev); @@ -4291,11 +4538,18 @@ static ssize_t wm_latency_write(struct file *file, const char __user *ubuf, struct seq_file *m = file->private_data; struct drm_device *dev = m->private; uint16_t new[8] = { 0 }; - int num_levels = ilk_wm_max_level(dev) + 1; + int num_levels; int level; int ret; char tmp[32]; + if (IS_CHERRYVIEW(dev)) + num_levels = 3; + else if (IS_VALLEYVIEW(dev)) + num_levels = 1; + else + num_levels = ilk_wm_max_level(dev) + 1; + if (len >= sizeof(tmp)) return -EINVAL; @@ -4796,7 +5050,7 @@ static void cherryview_sseu_device_status(struct drm_device *dev, struct sseu_dev_status *stat) { struct drm_i915_private *dev_priv = dev->dev_private; - const int ss_max = 2; + int ss_max = 2; int ss; u32 sig1[ss_max], sig2[ss_max]; @@ -4862,7 +5116,7 @@ static void gen9_sseu_device_status(struct drm_device *dev, stat->slice_total++; - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) ss_cnt = INTEL_INFO(dev)->subslice_per_slice; for (ss = 0; ss < ss_max; ss++) { @@ -4889,13 +5143,38 @@ static void gen9_sseu_device_status(struct drm_device *dev, } } +static void broadwell_sseu_device_status(struct drm_device *dev, + struct sseu_dev_status *stat) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int s; + u32 slice_info = I915_READ(GEN8_GT_SLICE_INFO); + + stat->slice_total = hweight32(slice_info & GEN8_LSLICESTAT_MASK); + + if (stat->slice_total) { + stat->subslice_per_slice = INTEL_INFO(dev)->subslice_per_slice; + stat->subslice_total = stat->slice_total * + stat->subslice_per_slice; + stat->eu_per_subslice = INTEL_INFO(dev)->eu_per_subslice; + stat->eu_total = stat->eu_per_subslice * stat->subslice_total; + + /* subtract fused off EU(s) from enabled slice(s) */ + for (s = 0; s < stat->slice_total; s++) { + u8 subslice_7eu = INTEL_INFO(dev)->subslice_7eu[s]; + + stat->eu_total -= hweight8(subslice_7eu); + } + } +} + static int i915_sseu_status(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; struct sseu_dev_status stat; - if ((INTEL_INFO(dev)->gen < 8) || IS_BROADWELL(dev)) + if (INTEL_INFO(dev)->gen < 8) return -ENODEV; seq_puts(m, "SSEU Device Info\n"); @@ -4920,6 +5199,8 @@ static int i915_sseu_status(struct seq_file *m, void *unused) memset(&stat, 0, sizeof(stat)); if (IS_CHERRYVIEW(dev)) { cherryview_sseu_device_status(dev, &stat); + } else if (IS_BROADWELL(dev)) { + broadwell_sseu_device_status(dev, &stat); } else if (INTEL_INFO(dev)->gen >= 9) { gen9_sseu_device_status(dev, &stat); } @@ -5022,11 +5303,15 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS}, {"i915_gem_hws_vebox", i915_hws_info, 0, (void *)VECS}, {"i915_gem_batch_pool", i915_gem_batch_pool_info, 0}, + {"i915_guc_info", i915_guc_info, 0}, + {"i915_guc_load_status", i915_guc_load_status_info, 0}, + {"i915_guc_log_dump", i915_guc_log_dump, 0}, {"i915_frequency_info", i915_frequency_info, 0}, {"i915_hangcheck_info", i915_hangcheck_info, 0}, {"i915_drpc_info", i915_drpc_info, 0}, {"i915_emon_status", i915_emon_status, 0}, {"i915_ring_freq_table", i915_ring_freq_table, 0}, + {"i915_frontbuffer_tracking", i915_frontbuffer_tracking, 0}, {"i915_fbc_status", i915_fbc_status, 0}, {"i915_ips_status", i915_ips_status, 0}, {"i915_sr_status", i915_sr_status, 0}, @@ -5042,8 +5327,9 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_edp_psr_status", i915_edp_psr_status, 0}, {"i915_sink_crc_eDP1", i915_sink_crc, 0}, {"i915_energy_uJ", i915_energy_uJ, 0}, - {"i915_pc8_status", i915_pc8_status, 0}, + {"i915_runtime_pm_status", i915_runtime_pm_status, 0}, {"i915_power_domain_info", i915_power_domain_info, 0}, + {"i915_dmc_info", i915_dmc_info, 0}, {"i915_display_info", i915_display_info, 0}, {"i915_semaphore_status", i915_semaphore_status, 0}, {"i915_shared_dplls_info", i915_shared_dplls_info, 0}, diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c index d2df321ba634..a81c76603544 100644 --- a/drivers/gpu/drm/i915/i915_dma.c +++ b/drivers/gpu/drm/i915/i915_dma.c @@ -28,7 +28,6 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/async.h> #include <drm/drmP.h> #include <drm/drm_crtc_helper.h> #include <drm/drm_fb_helper.h> @@ -75,7 +74,7 @@ static int i915_getparam(struct drm_device *dev, void *data, value = 1; break; case I915_PARAM_NUM_FENCES_AVAIL: - value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; + value = dev_priv->num_fence_regs; break; case I915_PARAM_HAS_OVERLAY: value = dev_priv->overlay ? 1 : 0; @@ -163,6 +162,13 @@ static int i915_getparam(struct drm_device *dev, void *data, if (!value) return -ENODEV; break; + case I915_PARAM_HAS_GPU_RESET: + value = i915.enable_hangcheck && + intel_has_gpu_reset(dev); + break; + case I915_PARAM_HAS_RESOURCE_STREAMER: + value = HAS_RESOURCE_STREAMER(dev); + break; default: DRM_DEBUG("Unknown parameter %d\n", param->param); return -EINVAL; @@ -176,35 +182,6 @@ static int i915_getparam(struct drm_device *dev, void *data, return 0; } -static int i915_setparam(struct drm_device *dev, void *data, - struct drm_file *file_priv) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - drm_i915_setparam_t *param = data; - - switch (param->param) { - case I915_SETPARAM_USE_MI_BATCHBUFFER_START: - case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY: - case I915_SETPARAM_ALLOW_BATCHBUFFER: - /* Reject all old ums/dri params. */ - return -ENODEV; - - case I915_SETPARAM_NUM_USED_FENCES: - if (param->value > dev_priv->num_fence_regs || - param->value < 0) - return -EINVAL; - /* Userspace can use first N regs */ - dev_priv->fence_reg_start = param->value; - break; - default: - DRM_DEBUG_DRIVER("unknown parameter %d\n", - param->param); - return -EINVAL; - } - - return 0; -} - static int i915_get_bridge_dev(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -357,12 +334,12 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; /* i915 resume handler doesn't set to D0 */ pci_set_power_state(dev->pdev, PCI_D0); - i915_resume_legacy(dev); + i915_resume_switcheroo(dev); dev->switch_power_state = DRM_SWITCH_POWER_ON; } else { - pr_err("switched off\n"); + pr_info("switched off\n"); dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - i915_suspend_legacy(dev, pmm); + i915_suspend_switcheroo(dev, pmm); dev->switch_power_state = DRM_SWITCH_POWER_OFF; } } @@ -418,7 +395,9 @@ static int i915_load_modeset_init(struct drm_device *dev) if (ret) goto cleanup_vga_switcheroo; - intel_power_domains_init_hw(dev_priv); + intel_power_domains_init_hw(dev_priv, false); + + intel_csr_ucode_init(dev_priv); ret = intel_irq_install(dev_priv); if (ret) @@ -428,6 +407,8 @@ static int i915_load_modeset_init(struct drm_device *dev) * working irqs for e.g. gmbus and dp aux transfers. */ intel_modeset_init(dev); + intel_guc_ucode_init(dev); + ret = i915_gem_init(dev); if (ret) goto cleanup_irq; @@ -457,7 +438,7 @@ static int i915_load_modeset_init(struct drm_device *dev) * scanning against hotplug events. Hence do this first and ignore the * tiny window where we will loose hotplug notifactions. */ - async_schedule(intel_fbdev_initial_config, dev_priv); + intel_fbdev_initial_config_async(dev); drm_kms_helper_poll_init(dev); @@ -469,6 +450,7 @@ cleanup_gem: i915_gem_context_fini(dev); mutex_unlock(&dev->struct_mutex); cleanup_irq: + intel_guc_ucode_fini(dev); drm_irq_uninstall(dev); cleanup_gem_stolen: i915_gem_cleanup_stolen(dev); @@ -616,17 +598,6 @@ static void gen9_sseu_info_init(struct drm_device *dev) u32 fuse2, s_enable, ss_disable, eu_disable; u8 eu_mask = 0xff; - /* - * BXT has a single slice. BXT also has at most 6 EU per subslice, - * and therefore only the lowest 6 bits of the 8-bit EU disable - * fields are valid. - */ - if (IS_BROXTON(dev)) { - s_max = 1; - eu_max = 6; - eu_mask = 0x3f; - } - info = (struct intel_device_info *)&dev_priv->info; fuse2 = I915_READ(GEN8_FUSE2); s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> @@ -693,11 +664,88 @@ static void gen9_sseu_info_init(struct drm_device *dev) * supports EU power gating on devices with more than one EU * pair per subslice. */ - info->has_slice_pg = (IS_SKYLAKE(dev) && (info->slice_total > 1)); + info->has_slice_pg = ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) && + (info->slice_total > 1)); info->has_subslice_pg = (IS_BROXTON(dev) && (info->subslice_total > 1)); info->has_eu_pg = (info->eu_per_subslice > 2); } +static void broadwell_sseu_info_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_device_info *info; + const int s_max = 3, ss_max = 3, eu_max = 8; + int s, ss; + u32 fuse2, eu_disable[s_max], s_enable, ss_disable; + + fuse2 = I915_READ(GEN8_FUSE2); + s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; + ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT; + + eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK; + eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) | + ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) << + (32 - GEN8_EU_DIS0_S1_SHIFT)); + eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) | + ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) << + (32 - GEN8_EU_DIS1_S2_SHIFT)); + + + info = (struct intel_device_info *)&dev_priv->info; + info->slice_total = hweight32(s_enable); + + /* + * The subslice disable field is global, i.e. it applies + * to each of the enabled slices. + */ + info->subslice_per_slice = ss_max - hweight32(ss_disable); + info->subslice_total = info->slice_total * info->subslice_per_slice; + + /* + * Iterate through enabled slices and subslices to + * count the total enabled EU. + */ + for (s = 0; s < s_max; s++) { + if (!(s_enable & (0x1 << s))) + /* skip disabled slice */ + continue; + + for (ss = 0; ss < ss_max; ss++) { + u32 n_disabled; + + if (ss_disable & (0x1 << ss)) + /* skip disabled subslice */ + continue; + + n_disabled = hweight8(eu_disable[s] >> (ss * eu_max)); + + /* + * Record which subslices have 7 EUs. + */ + if (eu_max - n_disabled == 7) + info->subslice_7eu[s] |= 1 << ss; + + info->eu_total += eu_max - n_disabled; + } + } + + /* + * BDW is expected to always have a uniform distribution of EU across + * subslices with the exception that any one EU in any one subslice may + * be fused off for die recovery. + */ + info->eu_per_subslice = info->subslice_total ? + DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0; + + /* + * BDW supports slice power gating on devices with more than + * one slice. + */ + info->has_slice_pg = (info->slice_total > 1); + info->has_subslice_pg = 0; + info->has_eu_pg = 0; +} + /* * Determine various intel_device_info fields at runtime. * @@ -719,11 +767,19 @@ static void intel_device_info_runtime_init(struct drm_device *dev) info = (struct intel_device_info *)&dev_priv->info; + /* + * Skylake and Broxton currently don't expose the topmost plane as its + * use is exclusive with the legacy cursor and we only want to expose + * one of those, not both. Until we can safely expose the topmost plane + * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported, + * we don't expose the topmost plane at all to prevent ABI breakage + * down the line. + */ if (IS_BROXTON(dev)) { - info->num_sprites[PIPE_A] = 3; - info->num_sprites[PIPE_B] = 3; - info->num_sprites[PIPE_C] = 2; - } else if (IS_VALLEYVIEW(dev) || INTEL_INFO(dev)->gen == 9) + info->num_sprites[PIPE_A] = 2; + info->num_sprites[PIPE_B] = 2; + info->num_sprites[PIPE_C] = 1; + } else if (IS_VALLEYVIEW(dev)) for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 2; else @@ -760,6 +816,8 @@ static void intel_device_info_runtime_init(struct drm_device *dev) /* Initialize slice/subslice/EU info */ if (IS_CHERRYVIEW(dev)) cherryview_sseu_info_init(dev); + else if (IS_BROADWELL(dev)) + broadwell_sseu_info_init(dev); else if (INTEL_INFO(dev)->gen >= 9) gen9_sseu_info_init(dev); @@ -776,6 +834,24 @@ static void intel_device_info_runtime_init(struct drm_device *dev) info->has_eu_pg ? "y" : "n"); } +static void intel_init_dpio(struct drm_i915_private *dev_priv) +{ + if (!IS_VALLEYVIEW(dev_priv)) + return; + + /* + * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), + * CHV x1 PHY (DP/HDMI D) + * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) + */ + if (IS_CHERRYVIEW(dev_priv)) { + DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; + DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; + } else { + DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; + } +} + /** * i915_driver_load - setup chip and create an initial config * @dev: DRM device @@ -816,7 +892,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) spin_lock_init(&dev_priv->mmio_flip_lock); mutex_init(&dev_priv->sb_lock); mutex_init(&dev_priv->modeset_restore_lock); - mutex_init(&dev_priv->csr_lock); + mutex_init(&dev_priv->av_mutex); intel_pm_setup(dev); @@ -862,9 +938,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) intel_uncore_init(dev); - /* Load CSR Firmware for SKL */ - intel_csr_ucode_init(dev); - ret = i915_gem_gtt_init(dev); if (ret) goto out_freecsr; @@ -933,8 +1006,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) goto out_mtrrfree; } - dev_priv->dp_wq = alloc_ordered_workqueue("i915-dp", 0); - if (dev_priv->dp_wq == NULL) { + dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); + if (dev_priv->hotplug.dp_wq == NULL) { DRM_ERROR("Failed to create our dp workqueue.\n"); ret = -ENOMEM; goto out_freewq; @@ -956,8 +1029,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) intel_setup_gmbus(dev); intel_opregion_setup(dev); - intel_setup_bios(dev); - i915_gem_load(dev); /* On the 945G/GM, the chipset reports the MSI capability on the @@ -976,6 +1047,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) intel_device_info_runtime_init(dev); + intel_init_dpio(dev_priv); + if (INTEL_INFO(dev)->num_pipes) { ret = drm_vblank_init(dev, INTEL_INFO(dev)->num_pipes); if (ret) @@ -1029,7 +1102,7 @@ out_gem_unload: pm_qos_remove_request(&dev_priv->pm_qos); destroy_workqueue(dev_priv->gpu_error.hangcheck_wq); out_freedpwq: - destroy_workqueue(dev_priv->dp_wq); + destroy_workqueue(dev_priv->hotplug.dp_wq); out_freewq: destroy_workqueue(dev_priv->wq); out_mtrrfree: @@ -1038,18 +1111,15 @@ out_mtrrfree: out_gtt: i915_global_gtt_cleanup(dev); out_freecsr: - intel_csr_ucode_fini(dev); + intel_csr_ucode_fini(dev_priv); intel_uncore_fini(dev); pci_iounmap(dev->pdev, dev_priv->regs); put_bridge: pci_dev_put(dev_priv->bridge_dev); free_priv: - if (dev_priv->requests) - kmem_cache_destroy(dev_priv->requests); - if (dev_priv->vmas) - kmem_cache_destroy(dev_priv->vmas); - if (dev_priv->objects) - kmem_cache_destroy(dev_priv->objects); + kmem_cache_destroy(dev_priv->requests); + kmem_cache_destroy(dev_priv->vmas); + kmem_cache_destroy(dev_priv->objects); kfree(dev_priv); return ret; } @@ -1059,6 +1129,8 @@ int i915_driver_unload(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int ret; + intel_fbdev_fini(dev); + i915_audio_component_cleanup(dev_priv); ret = i915_gem_suspend(dev); @@ -1081,8 +1153,6 @@ int i915_driver_unload(struct drm_device *dev) acpi_video_unregister(); - intel_fbdev_fini(dev); - drm_vblank_cleanup(dev); intel_modeset_cleanup(dev); @@ -1096,6 +1166,10 @@ int i915_driver_unload(struct drm_device *dev) dev_priv->vbt.child_dev = NULL; dev_priv->vbt.child_dev_num = 0; } + kfree(dev_priv->vbt.sdvo_lvds_vbt_mode); + dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; + kfree(dev_priv->vbt.lfp_lvds_vbt_mode); + dev_priv->vbt.lfp_lvds_vbt_mode = NULL; vga_switcheroo_unregister_client(dev->pdev); vga_client_register(dev->pdev, NULL, NULL, NULL); @@ -1112,18 +1186,20 @@ int i915_driver_unload(struct drm_device *dev) /* Flush any outstanding unpin_work. */ flush_workqueue(dev_priv->wq); + intel_guc_ucode_fini(dev); mutex_lock(&dev->struct_mutex); i915_gem_cleanup_ringbuffer(dev); i915_gem_context_fini(dev); mutex_unlock(&dev->struct_mutex); + intel_fbc_cleanup_cfb(dev_priv); i915_gem_cleanup_stolen(dev); - intel_csr_ucode_fini(dev); + intel_csr_ucode_fini(dev_priv); intel_teardown_gmbus(dev); intel_teardown_mchbar(dev); - destroy_workqueue(dev_priv->dp_wq); + destroy_workqueue(dev_priv->hotplug.dp_wq); destroy_workqueue(dev_priv->wq); destroy_workqueue(dev_priv->gpu_error.hangcheck_wq); pm_qos_remove_request(&dev_priv->pm_qos); @@ -1134,13 +1210,9 @@ int i915_driver_unload(struct drm_device *dev) if (dev_priv->regs != NULL) pci_iounmap(dev->pdev, dev_priv->regs); - if (dev_priv->requests) - kmem_cache_destroy(dev_priv->requests); - if (dev_priv->vmas) - kmem_cache_destroy(dev_priv->vmas); - if (dev_priv->objects) - kmem_cache_destroy(dev_priv->objects); - + kmem_cache_destroy(dev_priv->requests); + kmem_cache_destroy(dev_priv->vmas); + kmem_cache_destroy(dev_priv->objects); pci_dev_put(dev_priv->bridge_dev); kfree(dev_priv); @@ -1190,8 +1262,6 @@ void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; - if (file_priv && file_priv->bsd_ring) - file_priv->bsd_ring = NULL; kfree(file_priv); } @@ -1210,7 +1280,7 @@ const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), @@ -1220,51 +1290,41 @@ const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), - DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED), - DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0), + DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), }; int i915_max_ioctl = ARRAY_SIZE(i915_ioctls); - -/* - * This is really ugly: Because old userspace abused the linux agp interface to - * manage the gtt, we need to claim that all intel devices are agp. For - * otherwise the drm core refuses to initialize the agp support code. - */ -int i915_driver_device_is_agp(struct drm_device *dev) -{ - return 1; -} diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 884b4f9b81c4..e6935f1cb689 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -228,121 +228,83 @@ static const struct intel_device_info intel_sandybridge_m_info = { .need_gfx_hws = 1, .has_hotplug = 1, \ .has_fbc = 1, \ .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ - .has_llc = 1 + .has_llc = 1, \ + GEN_DEFAULT_PIPEOFFSETS, \ + IVB_CURSOR_OFFSETS static const struct intel_device_info intel_ivybridge_d_info = { GEN7_FEATURES, .is_ivybridge = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_ivybridge_m_info = { GEN7_FEATURES, .is_ivybridge = 1, .is_mobile = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_ivybridge_q_info = { GEN7_FEATURES, .is_ivybridge = 1, .num_pipes = 0, /* legal, last one wins */ - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; +#define VLV_FEATURES \ + .gen = 7, .num_pipes = 2, \ + .need_gfx_hws = 1, .has_hotplug = 1, \ + .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ + .display_mmio_offset = VLV_DISPLAY_BASE, \ + GEN_DEFAULT_PIPEOFFSETS, \ + CURSOR_OFFSETS + static const struct intel_device_info intel_valleyview_m_info = { - GEN7_FEATURES, - .is_mobile = 1, - .num_pipes = 2, + VLV_FEATURES, .is_valleyview = 1, - .display_mmio_offset = VLV_DISPLAY_BASE, - .has_fbc = 0, /* legal, last one wins */ - .has_llc = 0, /* legal, last one wins */ - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, + .is_mobile = 1, }; static const struct intel_device_info intel_valleyview_d_info = { - GEN7_FEATURES, - .num_pipes = 2, + VLV_FEATURES, .is_valleyview = 1, - .display_mmio_offset = VLV_DISPLAY_BASE, - .has_fbc = 0, /* legal, last one wins */ - .has_llc = 0, /* legal, last one wins */ - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, }; +#define HSW_FEATURES \ + GEN7_FEATURES, \ + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \ + .has_ddi = 1, \ + .has_fpga_dbg = 1 + static const struct intel_device_info intel_haswell_d_info = { - GEN7_FEATURES, + HSW_FEATURES, .is_haswell = 1, - .has_ddi = 1, - .has_fpga_dbg = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_haswell_m_info = { - GEN7_FEATURES, + HSW_FEATURES, .is_haswell = 1, .is_mobile = 1, - .has_ddi = 1, - .has_fpga_dbg = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broadwell_d_info = { - .gen = 8, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - .has_llc = 1, - .has_ddi = 1, - .has_fpga_dbg = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, + HSW_FEATURES, + .gen = 8, }; static const struct intel_device_info intel_broadwell_m_info = { - .gen = 8, .is_mobile = 1, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - .has_llc = 1, - .has_ddi = 1, - .has_fpga_dbg = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, + HSW_FEATURES, + .gen = 8, .is_mobile = 1, }; static const struct intel_device_info intel_broadwell_gt3d_info = { - .gen = 8, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, + HSW_FEATURES, + .gen = 8, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, - .has_llc = 1, - .has_ddi = 1, - .has_fpga_dbg = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broadwell_gt3m_info = { - .gen = 8, .is_mobile = 1, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, + HSW_FEATURES, + .gen = 8, .is_mobile = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, - .has_llc = 1, - .has_ddi = 1, - .has_fpga_dbg = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_cherryview_info = { @@ -356,93 +318,125 @@ static const struct intel_device_info intel_cherryview_info = { }; static const struct intel_device_info intel_skylake_info = { - .is_preliminary = 1, + HSW_FEATURES, .is_skylake = 1, - .gen = 9, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - .has_llc = 1, - .has_ddi = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, + .gen = 9, }; static const struct intel_device_info intel_skylake_gt3_info = { - .is_preliminary = 1, + HSW_FEATURES, .is_skylake = 1, - .gen = 9, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, + .gen = 9, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, - .has_llc = 1, - .has_ddi = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broxton_info = { .is_preliminary = 1, + .is_broxton = 1, .gen = 9, .need_gfx_hws = 1, .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .num_pipes = 3, .has_ddi = 1, + .has_fpga_dbg = 1, .has_fbc = 1, GEN_DEFAULT_PIPEOFFSETS, IVB_CURSOR_OFFSETS, }; +static const struct intel_device_info intel_kabylake_info = { + HSW_FEATURES, + .is_preliminary = 1, + .is_kabylake = 1, + .gen = 9, +}; + +static const struct intel_device_info intel_kabylake_gt3_info = { + HSW_FEATURES, + .is_preliminary = 1, + .is_kabylake = 1, + .gen = 9, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, +}; + /* * Make sure any device matches here are from most specific to most * general. For example, since the Quanta match is based on the subsystem * and subvendor IDs, we need it to come before the more general IVB * PCI ID matches, otherwise we'll use the wrong info struct above. */ -#define INTEL_PCI_IDS \ - INTEL_I830_IDS(&intel_i830_info), \ - INTEL_I845G_IDS(&intel_845g_info), \ - INTEL_I85X_IDS(&intel_i85x_info), \ - INTEL_I865G_IDS(&intel_i865g_info), \ - INTEL_I915G_IDS(&intel_i915g_info), \ - INTEL_I915GM_IDS(&intel_i915gm_info), \ - INTEL_I945G_IDS(&intel_i945g_info), \ - INTEL_I945GM_IDS(&intel_i945gm_info), \ - INTEL_I965G_IDS(&intel_i965g_info), \ - INTEL_G33_IDS(&intel_g33_info), \ - INTEL_I965GM_IDS(&intel_i965gm_info), \ - INTEL_GM45_IDS(&intel_gm45_info), \ - INTEL_G45_IDS(&intel_g45_info), \ - INTEL_PINEVIEW_IDS(&intel_pineview_info), \ - INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info), \ - INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info), \ - INTEL_SNB_D_IDS(&intel_sandybridge_d_info), \ - INTEL_SNB_M_IDS(&intel_sandybridge_m_info), \ - INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ \ - INTEL_IVB_M_IDS(&intel_ivybridge_m_info), \ - INTEL_IVB_D_IDS(&intel_ivybridge_d_info), \ - INTEL_HSW_D_IDS(&intel_haswell_d_info), \ - INTEL_HSW_M_IDS(&intel_haswell_m_info), \ - INTEL_VLV_M_IDS(&intel_valleyview_m_info), \ - INTEL_VLV_D_IDS(&intel_valleyview_d_info), \ - INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info), \ - INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info), \ - INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), \ - INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), \ - INTEL_CHV_IDS(&intel_cherryview_info), \ - INTEL_SKL_GT1_IDS(&intel_skylake_info), \ - INTEL_SKL_GT2_IDS(&intel_skylake_info), \ - INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info), \ - INTEL_BXT_IDS(&intel_broxton_info) - -static const struct pci_device_id pciidlist[] = { /* aka */ - INTEL_PCI_IDS, +static const struct pci_device_id pciidlist[] = { + INTEL_I830_IDS(&intel_i830_info), + INTEL_I845G_IDS(&intel_845g_info), + INTEL_I85X_IDS(&intel_i85x_info), + INTEL_I865G_IDS(&intel_i865g_info), + INTEL_I915G_IDS(&intel_i915g_info), + INTEL_I915GM_IDS(&intel_i915gm_info), + INTEL_I945G_IDS(&intel_i945g_info), + INTEL_I945GM_IDS(&intel_i945gm_info), + INTEL_I965G_IDS(&intel_i965g_info), + INTEL_G33_IDS(&intel_g33_info), + INTEL_I965GM_IDS(&intel_i965gm_info), + INTEL_GM45_IDS(&intel_gm45_info), + INTEL_G45_IDS(&intel_g45_info), + INTEL_PINEVIEW_IDS(&intel_pineview_info), + INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info), + INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info), + INTEL_SNB_D_IDS(&intel_sandybridge_d_info), + INTEL_SNB_M_IDS(&intel_sandybridge_m_info), + INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ + INTEL_IVB_M_IDS(&intel_ivybridge_m_info), + INTEL_IVB_D_IDS(&intel_ivybridge_d_info), + INTEL_HSW_D_IDS(&intel_haswell_d_info), + INTEL_HSW_M_IDS(&intel_haswell_m_info), + INTEL_VLV_M_IDS(&intel_valleyview_m_info), + INTEL_VLV_D_IDS(&intel_valleyview_d_info), + INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info), + INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info), + INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), + INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), + INTEL_CHV_IDS(&intel_cherryview_info), + INTEL_SKL_GT1_IDS(&intel_skylake_info), + INTEL_SKL_GT2_IDS(&intel_skylake_info), + INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info), + INTEL_SKL_GT4_IDS(&intel_skylake_gt3_info), + INTEL_BXT_IDS(&intel_broxton_info), + INTEL_KBL_GT1_IDS(&intel_kabylake_info), + INTEL_KBL_GT2_IDS(&intel_kabylake_info), + INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info), + INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info), {0, 0, 0} }; -#if defined(CONFIG_DRM_I915_KMS) MODULE_DEVICE_TABLE(pci, pciidlist); -#endif + +static enum intel_pch intel_virt_detect_pch(struct drm_device *dev) +{ + enum intel_pch ret = PCH_NOP; + + /* + * In a virtualized passthrough environment we can be in a + * setup where the ISA bridge is not able to be passed through. + * In this case, a south bridge can be emulated and we have to + * make an educated guess as to which PCH is really there. + */ + + if (IS_GEN5(dev)) { + ret = PCH_IBX; + DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n"); + } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) { + ret = PCH_CPT; + DRM_DEBUG_KMS("Assuming CouarPoint PCH\n"); + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + ret = PCH_LPT; + DRM_DEBUG_KMS("Assuming LynxPoint PCH\n"); + } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + ret = PCH_SPT; + DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n"); + } + + return ret; +} void intel_detect_pch(struct drm_device *dev) { @@ -499,11 +493,16 @@ void intel_detect_pch(struct drm_device *dev) } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) { dev_priv->pch_type = PCH_SPT; DRM_DEBUG_KMS("Found SunrisePoint PCH\n"); - WARN_ON(!IS_SKYLAKE(dev)); + WARN_ON(!IS_SKYLAKE(dev) && + !IS_KABYLAKE(dev)); } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) { dev_priv->pch_type = PCH_SPT; DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n"); - WARN_ON(!IS_SKYLAKE(dev)); + WARN_ON(!IS_SKYLAKE(dev) && + !IS_KABYLAKE(dev)); + } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) || + (id == INTEL_PCH_QEMU_DEVICE_ID_TYPE)) { + dev_priv->pch_type = intel_virt_detect_pch(dev); } else continue; @@ -541,41 +540,6 @@ bool i915_semaphore_is_enabled(struct drm_device *dev) return true; } -void intel_hpd_cancel_work(struct drm_i915_private *dev_priv) -{ - spin_lock_irq(&dev_priv->irq_lock); - - dev_priv->long_hpd_port_mask = 0; - dev_priv->short_hpd_port_mask = 0; - dev_priv->hpd_event_bits = 0; - - spin_unlock_irq(&dev_priv->irq_lock); - - cancel_work_sync(&dev_priv->dig_port_work); - cancel_work_sync(&dev_priv->hotplug_work); - cancel_delayed_work_sync(&dev_priv->hotplug_reenable_work); -} - -void i915_firmware_load_error_print(const char *fw_path, int err) -{ - DRM_ERROR("failed to load firmware %s (%d)\n", fw_path, err); - - /* - * If the reason is not known assume -ENOENT since that's the most - * usual failure mode. - */ - if (!err) - err = -ENOENT; - - if (!(IS_BUILTIN(CONFIG_DRM_I915) && err == -ENOENT)) - return; - - DRM_ERROR( - "The driver is built-in, so to load the firmware you need to\n" - "include it either in the kernel (see CONFIG_EXTRA_FIRMWARE) or\n" - "in your initrd/initramfs image.\n"); -} - static void intel_suspend_encoders(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; @@ -594,14 +558,20 @@ static void intel_suspend_encoders(struct drm_i915_private *dev_priv) static int intel_suspend_complete(struct drm_i915_private *dev_priv); static int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume); -static int skl_resume_prepare(struct drm_i915_private *dev_priv); static int bxt_resume_prepare(struct drm_i915_private *dev_priv); +static bool suspend_to_idle(struct drm_i915_private *dev_priv) +{ +#if IS_ENABLED(CONFIG_ACPI_SLEEP) + if (acpi_target_system_state() < ACPI_STATE_S3) + return true; +#endif + return false; +} static int i915_drm_suspend(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_crtc *crtc; pci_power_t opregion_target_state; int error; @@ -625,6 +595,8 @@ static int i915_drm_suspend(struct drm_device *dev) return error; } + intel_guc_suspend(dev); + intel_suspend_gt_powersave(dev); /* @@ -632,8 +604,7 @@ static int i915_drm_suspend(struct drm_device *dev) * for _thaw. Also, power gate the CRTC power wells. */ drm_modeset_lock_all(dev); - for_each_crtc(dev, crtc) - intel_crtc_control(crtc, false); + intel_display_suspend(dev); drm_modeset_unlock_all(dev); intel_dp_mst_suspend(dev); @@ -649,11 +620,7 @@ static int i915_drm_suspend(struct drm_device *dev) i915_save_state(dev); - opregion_target_state = PCI_D3cold; -#if IS_ENABLED(CONFIG_ACPI_SLEEP) - if (acpi_target_system_state() < ACPI_STATE_S3) - opregion_target_state = PCI_D1; -#endif + opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; intel_opregion_notify_adapter(dev, opregion_target_state); intel_uncore_forcewake_reset(dev, false); @@ -665,39 +632,61 @@ static int i915_drm_suspend(struct drm_device *dev) intel_display_set_init_power(dev_priv, false); + if (HAS_CSR(dev_priv)) + flush_work(&dev_priv->csr.work); + return 0; } static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation) { struct drm_i915_private *dev_priv = drm_dev->dev_private; + bool fw_csr; int ret; + fw_csr = suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload; + /* + * In case of firmware assisted context save/restore don't manually + * deinit the power domains. This also means the CSR/DMC firmware will + * stay active, it will power down any HW resources as required and + * also enable deeper system power states that would be blocked if the + * firmware was inactive. + */ + if (!fw_csr) + intel_power_domains_suspend(dev_priv); + ret = intel_suspend_complete(dev_priv); if (ret) { DRM_ERROR("Suspend complete failed: %d\n", ret); + if (!fw_csr) + intel_power_domains_init_hw(dev_priv, true); return ret; } pci_disable_device(drm_dev->pdev); /* - * During hibernation on some GEN4 platforms the BIOS may try to access + * During hibernation on some platforms the BIOS may try to access * the device even though it's already in D3 and hang the machine. So * leave the device in D0 on those platforms and hope the BIOS will - * power down the device properly. Platforms where this was seen: - * Lenovo Thinkpad X301, X61s + * power down the device properly. The issue was seen on multiple old + * GENs with different BIOS vendors, so having an explicit blacklist + * is inpractical; apply the workaround on everything pre GEN6. The + * platforms where the issue was seen: + * Lenovo Thinkpad X301, X61s, X60, T60, X41 + * Fujitsu FSC S7110 + * Acer Aspire 1830T */ - if (!(hibernation && - drm_dev->pdev->subsystem_vendor == PCI_VENDOR_ID_LENOVO && - INTEL_INFO(dev_priv)->gen == 4)) + if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6)) pci_set_power_state(drm_dev->pdev, PCI_D3hot); + dev_priv->suspended_to_idle = suspend_to_idle(dev_priv); + return 0; } -int i915_suspend_legacy(struct drm_device *dev, pm_message_t state) +int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state) { int error; @@ -748,10 +737,12 @@ static int i915_drm_resume(struct drm_device *dev) mutex_lock(&dev->struct_mutex); if (i915_gem_init_hw(dev)) { DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); - atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); } mutex_unlock(&dev->struct_mutex); + intel_guc_resume(dev); + intel_modeset_init_hw(dev); spin_lock_irq(&dev_priv->irq_lock); @@ -760,7 +751,7 @@ static int i915_drm_resume(struct drm_device *dev) spin_unlock_irq(&dev_priv->irq_lock); drm_modeset_lock_all(dev); - intel_modeset_setup_hw_state(dev, true); + intel_display_resume(dev); drm_modeset_unlock_all(dev); intel_dp_mst_resume(dev); @@ -804,8 +795,10 @@ static int i915_drm_resume_early(struct drm_device *dev) * FIXME: This should be solved with a special hdmi sink device or * similar so that power domains can be employed. */ - if (pci_enable_device(dev->pdev)) - return -EIO; + if (pci_enable_device(dev->pdev)) { + ret = -EIO; + goto out; + } pci_set_master(dev->pdev); @@ -819,18 +812,21 @@ static int i915_drm_resume_early(struct drm_device *dev) if (IS_BROXTON(dev)) ret = bxt_resume_prepare(dev_priv); - else if (IS_SKYLAKE(dev_priv)) - ret = skl_resume_prepare(dev_priv); else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) hsw_disable_pc8(dev_priv); intel_uncore_sanitize(dev); - intel_power_domains_init_hw(dev_priv); + + if (!(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload)) + intel_power_domains_init_hw(dev_priv, true); + +out: + dev_priv->suspended_to_idle = false; return ret; } -int i915_resume_legacy(struct drm_device *dev) +int i915_resume_switcheroo(struct drm_device *dev) { int ret; @@ -865,9 +861,6 @@ int i915_reset(struct drm_device *dev) bool simulated; int ret; - if (!i915.reset) - return 0; - intel_reset_gt_powersave(dev); mutex_lock(&dev->struct_mutex); @@ -959,8 +952,6 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (PCI_FUNC(pdev->devfn)) return -ENODEV; - driver.driver_features &= ~(DRIVER_USE_AGP); - return drm_get_pci_dev(pdev, ent, &driver); } @@ -1037,21 +1028,6 @@ static int i915_pm_resume(struct device *dev) return i915_drm_resume(drm_dev); } -static int skl_suspend_complete(struct drm_i915_private *dev_priv) -{ - /* Enabling DC6 is not a hard requirement to enter runtime D3 */ - - /* - * This is to ensure that CSR isn't identified as loaded before - * CSR-loading program is called during runtime-resume. - */ - intel_csr_load_status_set(dev_priv, FW_UNINITIALIZED); - - skl_uninit_cdclk(dev_priv); - - return 0; -} - static int hsw_suspend_complete(struct drm_i915_private *dev_priv) { hsw_enable_pc8(dev_priv); @@ -1091,16 +1067,6 @@ static int bxt_resume_prepare(struct drm_i915_private *dev_priv) return 0; } -static int skl_resume_prepare(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - - skl_init_cdclk(dev_priv); - intel_csr_load_program(dev); - - return 0; -} - /* * Save all Gunit registers that may be lost after a D3 and a subsequent * S0i[R123] transition. The list of registers needing a save/restore is @@ -1140,7 +1106,7 @@ static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) - s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS_BASE + i * 4); + s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); @@ -1184,7 +1150,7 @@ static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) s->pm_ier = I915_READ(GEN6_PMIER); for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) - s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH_BASE + i * 4); + s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); /* GT SA CZ domain, 0x100000-0x138124 */ s->tilectl = I915_READ(TILECTL); @@ -1222,7 +1188,7 @@ static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) - I915_WRITE(GEN7_LRA_LIMITS_BASE + i * 4, s->lra_limits[i]); + I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); @@ -1266,7 +1232,7 @@ static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) I915_WRITE(GEN6_PMIER, s->pm_ier); for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) - I915_WRITE(GEN7_GT_SCRATCH_BASE + i * 4, s->gt_scratch[i]); + I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); /* GT SA CZ domain, 0x100000-0x138124 */ I915_WRITE(TILECTL, s->tilectl); @@ -1496,6 +1462,8 @@ static int intel_runtime_suspend(struct device *device) i915_gem_release_all_mmaps(dev_priv); mutex_unlock(&dev->struct_mutex); + intel_guc_suspend(dev); + intel_suspend_gt_powersave(dev); intel_runtime_pm_disable_interrupts(dev_priv); @@ -1515,7 +1483,15 @@ static int intel_runtime_suspend(struct device *device) * FIXME: We really should find a document that references the arguments * used below! */ - if (IS_HASWELL(dev)) { + if (IS_BROADWELL(dev)) { + /* + * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop + * being detected, and the call we do at intel_runtime_resume() + * won't be able to restore them. Since PCI_D3hot matches the + * actual specification and appears to be working, use it. + */ + intel_opregion_notify_adapter(dev, PCI_D3hot); + } else { /* * current versions of firmware which depend on this opregion * notification have repurposed the D1 definition to mean @@ -1524,16 +1500,6 @@ static int intel_runtime_suspend(struct device *device) * the suspend path. */ intel_opregion_notify_adapter(dev, PCI_D1); - } else { - /* - * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop - * being detected, and the call we do at intel_runtime_resume() - * won't be able to restore them. Since PCI_D3hot matches the - * actual specification and appears to be working, use it. Let's - * assume the other non-Haswell platforms will stay the same as - * Broadwell. - */ - intel_opregion_notify_adapter(dev, PCI_D3hot); } assert_forcewakes_inactive(dev_priv); @@ -1557,13 +1523,13 @@ static int intel_runtime_resume(struct device *device) intel_opregion_notify_adapter(dev, PCI_D0); dev_priv->pm.suspended = false; + intel_guc_resume(dev); + if (IS_GEN6(dev_priv)) intel_init_pch_refclk(dev); if (IS_BROXTON(dev)) ret = bxt_resume_prepare(dev_priv); - else if (IS_SKYLAKE(dev)) - ret = skl_resume_prepare(dev_priv); else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) hsw_disable_pc8(dev_priv); else if (IS_VALLEYVIEW(dev_priv)) @@ -1577,6 +1543,15 @@ static int intel_runtime_resume(struct device *device) gen6_update_ring_freq(dev); intel_runtime_pm_enable_interrupts(dev_priv); + + /* + * On VLV/CHV display interrupts are part of the display + * power well, so hpd is reinitialized from there. For + * everyone else do it here. + */ + if (!IS_VALLEYVIEW(dev_priv)) + intel_hpd_init(dev_priv); + intel_enable_gt_powersave(dev); if (ret) @@ -1597,8 +1572,6 @@ static int intel_suspend_complete(struct drm_i915_private *dev_priv) if (IS_BROXTON(dev_priv)) ret = bxt_suspend_complete(dev_priv); - else if (IS_SKYLAKE(dev_priv)) - ret = skl_suspend_complete(dev_priv); else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) ret = hsw_suspend_complete(dev_priv); else if (IS_VALLEYVIEW(dev_priv)) @@ -1673,9 +1646,8 @@ static struct drm_driver driver = { * deal with them for Intel hardware. */ .driver_features = - DRIVER_USE_AGP | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME | - DRIVER_RENDER, + DRIVER_RENDER | DRIVER_MODESET, .load = i915_driver_load, .unload = i915_driver_unload, .open = i915_driver_open, @@ -1684,11 +1656,6 @@ static struct drm_driver driver = { .postclose = i915_driver_postclose, .set_busid = drm_pci_set_busid, - /* Used in place of i915_pm_ops for non-DRIVER_MODESET */ - .suspend = i915_suspend_legacy, - .resume = i915_resume_legacy, - - .device_is_agp = i915_driver_device_is_agp, #if defined(CONFIG_DEBUG_FS) .debugfs_init = i915_debugfs_init, .debugfs_cleanup = i915_debugfs_cleanup, @@ -1727,20 +1694,13 @@ static int __init i915_init(void) driver.num_ioctls = i915_max_ioctl; /* - * If CONFIG_DRM_I915_KMS is set, default to KMS unless - * explicitly disabled with the module pararmeter. - * - * Otherwise, just follow the parameter (defaulting to off). - * - * Allow optional vga_text_mode_force boot option to override - * the default behavior. + * Enable KMS by default, unless explicitly overriden by + * either the i915.modeset prarameter or by the + * vga_text_mode_force boot option. */ -#if defined(CONFIG_DRM_I915_KMS) - if (i915.modeset != 0) - driver.driver_features |= DRIVER_MODESET; -#endif - if (i915.modeset == 1) - driver.driver_features |= DRIVER_MODESET; + + if (i915.modeset == 0) + driver.driver_features &= ~DRIVER_MODESET; #ifdef CONFIG_VGA_CONSOLE if (vgacon_text_force() && i915.modeset == -1) @@ -1748,17 +1708,11 @@ static int __init i915_init(void) #endif if (!(driver.driver_features & DRIVER_MODESET)) { - driver.get_vblank_timestamp = NULL; /* Silently fail loading to not upset userspace. */ DRM_DEBUG_DRIVER("KMS and UMS disabled.\n"); return 0; } - /* - * FIXME: Note that we're lying to the DRM core here so that we can get access - * to the atomic ioctl and the atomic properties. Only plane operations on - * a single CRTC will actually work. - */ if (i915.nuclear_pageflip) driver.driver_features |= DRIVER_ATOMIC; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index fd1de451c8c6..f1a8a53e9e30 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -50,13 +50,14 @@ #include <linux/intel-iommu.h> #include <linux/kref.h> #include <linux/pm_qos.h> +#include "intel_guc.h" /* General customization: */ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20150522" +#define DRIVER_DATE "20151204" #undef WARN_ON /* Many gcc seem to no see through this and fall over :( */ @@ -67,11 +68,11 @@ BUILD_BUG_ON(__i915_warn_cond); \ WARN(__i915_warn_cond, "WARN_ON(" #x ")"); }) #else -#define WARN_ON(x) WARN((x), "WARN_ON(" #x ")") +#define WARN_ON(x) WARN((x), "WARN_ON(%s)", #x ) #endif #undef WARN_ON_ONCE -#define WARN_ON_ONCE(x) WARN_ONCE((x), "WARN_ON_ONCE(" #x ")") +#define WARN_ON_ONCE(x) WARN_ONCE((x), "WARN_ON_ONCE(%s)", #x ) #define MISSING_CASE(x) WARN(1, "Missing switch case (%lu) in %s\n", \ (long) (x), __func__); @@ -105,6 +106,11 @@ unlikely(__ret_warn_on); \ }) +static inline const char *yesno(bool v) +{ + return v ? "yes" : "no"; +} + enum pipe { INVALID_PIPE = -1, PIPE_A = 0, @@ -125,17 +131,17 @@ enum transcoder { #define transcoder_name(t) ((t) + 'A') /* - * This is the maximum (across all platforms) number of planes (primary + - * sprites) that can be active at the same time on one pipe. - * - * This value doesn't count the cursor plane. + * I915_MAX_PLANES in the enum below is the maximum (across all platforms) + * number of planes per CRTC. Not all platforms really have this many planes, + * which means some arrays of size I915_MAX_PLANES may have unused entries + * between the topmost sprite plane and the cursor plane. */ -#define I915_MAX_PLANES 4 - enum plane { PLANE_A = 0, PLANE_B, PLANE_C, + PLANE_CURSOR, + I915_MAX_PLANES, }; #define plane_name(p) ((p) + 'A') @@ -174,14 +180,11 @@ enum intel_display_power_domain { POWER_DOMAIN_TRANSCODER_B, POWER_DOMAIN_TRANSCODER_C, POWER_DOMAIN_TRANSCODER_EDP, - POWER_DOMAIN_PORT_DDI_A_2_LANES, - POWER_DOMAIN_PORT_DDI_A_4_LANES, - POWER_DOMAIN_PORT_DDI_B_2_LANES, - POWER_DOMAIN_PORT_DDI_B_4_LANES, - POWER_DOMAIN_PORT_DDI_C_2_LANES, - POWER_DOMAIN_PORT_DDI_C_4_LANES, - POWER_DOMAIN_PORT_DDI_D_2_LANES, - POWER_DOMAIN_PORT_DDI_D_4_LANES, + POWER_DOMAIN_PORT_DDI_A_LANES, + POWER_DOMAIN_PORT_DDI_B_LANES, + POWER_DOMAIN_PORT_DDI_C_LANES, + POWER_DOMAIN_PORT_DDI_D_LANES, + POWER_DOMAIN_PORT_DDI_E_LANES, POWER_DOMAIN_PORT_DSI, POWER_DOMAIN_PORT_CRT, POWER_DOMAIN_PORT_OTHER, @@ -192,6 +195,8 @@ enum intel_display_power_domain { POWER_DOMAIN_AUX_B, POWER_DOMAIN_AUX_C, POWER_DOMAIN_AUX_D, + POWER_DOMAIN_GMBUS, + POWER_DOMAIN_MODESET, POWER_DOMAIN_INIT, POWER_DOMAIN_NUM, @@ -206,17 +211,51 @@ enum intel_display_power_domain { enum hpd_pin { HPD_NONE = 0, - HPD_PORT_A = HPD_NONE, /* PORT_A is internal */ HPD_TV = HPD_NONE, /* TV is known to be unreliable */ HPD_CRT, HPD_SDVO_B, HPD_SDVO_C, + HPD_PORT_A, HPD_PORT_B, HPD_PORT_C, HPD_PORT_D, + HPD_PORT_E, HPD_NUM_PINS }; +#define for_each_hpd_pin(__pin) \ + for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++) + +struct i915_hotplug { + struct work_struct hotplug_work; + + struct { + unsigned long last_jiffies; + int count; + enum { + HPD_ENABLED = 0, + HPD_DISABLED = 1, + HPD_MARK_DISABLED = 2 + } state; + } stats[HPD_NUM_PINS]; + u32 event_bits; + struct delayed_work reenable_work; + + struct intel_digital_port *irq_port[I915_MAX_PORTS]; + u32 long_port_mask; + u32 short_port_mask; + struct work_struct dig_port_work; + + /* + * if we get a HPD irq from DP and a HPD irq from non-DP + * the non-DP HPD could block the workqueue on a mode config + * mutex getting, that userspace may have taken. However + * userspace is waiting on the DP workqueue to run which is + * blocked behind the non-DP one. + */ + struct workqueue_struct *dp_wq; +}; + #define I915_GEM_GPU_DOMAINS \ (I915_GEM_DOMAIN_RENDER | \ I915_GEM_DOMAIN_SAMPLER | \ @@ -243,6 +282,12 @@ enum hpd_pin { &dev->mode_config.plane_list, \ base.head) +#define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) \ + list_for_each_entry(intel_plane, \ + &(dev)->mode_config.plane_list, \ + base.head) \ + for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe) + #define for_each_intel_crtc(dev, intel_crtc) \ list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) @@ -258,15 +303,15 @@ enum hpd_pin { #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \ - if ((intel_encoder)->base.crtc == (__crtc)) + for_each_if ((intel_encoder)->base.crtc == (__crtc)) #define for_each_connector_on_encoder(dev, __encoder, intel_connector) \ list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \ - if ((intel_connector)->base.encoder == (__encoder)) + for_each_if ((intel_connector)->base.encoder == (__encoder)) #define for_each_power_domain(domain, mask) \ for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ - if ((1 << (domain)) & (mask)) + for_each_if ((1 << (domain)) & (mask)) struct drm_i915_private; struct i915_mm_struct; @@ -304,6 +349,8 @@ enum intel_dpll_id { /* hsw/bdw */ DPLL_ID_WRPLL1 = 0, DPLL_ID_WRPLL2 = 1, + DPLL_ID_SPLL = 2, + /* skl */ DPLL_ID_SKL_DPLL1 = 0, DPLL_ID_SKL_DPLL2 = 1, @@ -320,6 +367,7 @@ struct intel_dpll_hw_state { /* hsw, bdw */ uint32_t wrpll; + uint32_t spll; /* skl */ /* @@ -333,7 +381,8 @@ struct intel_dpll_hw_state { uint32_t cfgcr1, cfgcr2; /* bxt */ - uint32_t ebb0, pll0, pll1, pll2, pll3, pll6, pll8, pll10, pcsdw12; + uint32_t ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, + pcsdw12; }; struct intel_shared_dpll_config { @@ -343,7 +392,6 @@ struct intel_shared_dpll_config { struct intel_shared_dpll { struct intel_shared_dpll_config config; - struct intel_shared_dpll_config *new_config; int active; /* count of number of active CRTCs (i.e. DPMS on) */ bool on; /* is the PLL actually active? Disabled during modeset */ @@ -403,14 +451,14 @@ struct opregion_swsci; struct opregion_asle; struct intel_opregion { - struct opregion_header __iomem *header; - struct opregion_acpi __iomem *acpi; - struct opregion_swsci __iomem *swsci; + struct opregion_header *header; + struct opregion_acpi *acpi; + struct opregion_swsci *swsci; u32 swsci_gbda_sub_functions; u32 swsci_sbcb_sub_functions; - struct opregion_asle __iomem *asle; - void __iomem *vbt; - u32 __iomem *lid_state; + struct opregion_asle *asle; + void *vbt; + u32 *lid_state; struct work_struct asle_work; }; #define OPREGION_SIZE (8*1024) @@ -445,6 +493,7 @@ struct drm_i915_error_state { struct timeval time; char error_msg[128]; + int iommu; u32 reset_count; u32 suspend_count; @@ -507,7 +556,7 @@ struct drm_i915_error_state { struct drm_i915_error_object { int page_count; - u32 gtt_offset; + u64 gtt_offset; u32 *pages[0]; } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; @@ -533,7 +582,7 @@ struct drm_i915_error_state { u32 size; u32 name; u32 rseqno[I915_NUM_RINGS], wseqno; - u32 gtt_offset; + u64 gtt_offset; u32 read_domains; u32 write_domain; s32 fence_reg:I915_MAX_NUM_FENCE_BITS; @@ -559,9 +608,6 @@ struct intel_limit; struct dpll; struct drm_i915_display_funcs { - bool (*fbc_enabled)(struct drm_device *dev); - void (*enable_fbc)(struct drm_crtc *crtc); - void (*disable_fbc)(struct drm_device *dev); int (*get_display_clock_speed)(struct drm_device *dev); int (*get_fifo_size)(struct drm_device *dev, int plane); /** @@ -582,12 +628,11 @@ struct drm_i915_display_funcs { int target, int refclk, struct dpll *match_clock, struct dpll *best_clock); + int (*compute_pipe_wm)(struct intel_crtc *crtc, + struct drm_atomic_state *state); void (*update_wm)(struct drm_crtc *crtc); - void (*update_sprite_wm)(struct drm_plane *plane, - struct drm_crtc *crtc, - uint32_t sprite_width, uint32_t sprite_height, - int pixel_size, bool enable, bool scaled); - void (*modeset_global_resources)(struct drm_atomic_state *state); + int (*modeset_calc_cdclk)(struct drm_atomic_state *state); + void (*modeset_commit_cdclk)(struct drm_atomic_state *state); /* Returns the active state of the crtc, and if the crtc is active, * fills out the pipe-config with the hw state. */ bool (*get_pipe_config)(struct intel_crtc *, @@ -598,17 +643,16 @@ struct drm_i915_display_funcs { struct intel_crtc_state *crtc_state); void (*crtc_enable)(struct drm_crtc *crtc); void (*crtc_disable)(struct drm_crtc *crtc); - void (*off)(struct drm_crtc *crtc); void (*audio_codec_enable)(struct drm_connector *connector, struct intel_encoder *encoder, - struct drm_display_mode *mode); + const struct drm_display_mode *adjusted_mode); void (*audio_codec_disable)(struct intel_encoder *encoder); void (*fdi_link_train)(struct drm_crtc *crtc); void (*init_clock_gating)(struct drm_device *dev); int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags); void (*update_primary_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb, @@ -619,13 +663,6 @@ struct drm_i915_display_funcs { /* render clock increase/decrease */ /* display clock increase/decrease */ /* pll clock increase/decrease */ - - int (*setup_backlight)(struct intel_connector *connector, enum pipe pipe); - uint32_t (*get_backlight)(struct intel_connector *connector); - void (*set_backlight)(struct intel_connector *connector, - uint32_t level); - void (*disable_backlight)(struct intel_connector *connector); - void (*enable_backlight)(struct intel_connector *connector); }; enum forcewake_domain_id { @@ -651,18 +688,18 @@ struct intel_uncore_funcs { void (*force_wake_put)(struct drm_i915_private *dev_priv, enum forcewake_domains domains); - uint8_t (*mmio_readb)(struct drm_i915_private *dev_priv, off_t offset, bool trace); - uint16_t (*mmio_readw)(struct drm_i915_private *dev_priv, off_t offset, bool trace); - uint32_t (*mmio_readl)(struct drm_i915_private *dev_priv, off_t offset, bool trace); - uint64_t (*mmio_readq)(struct drm_i915_private *dev_priv, off_t offset, bool trace); + uint8_t (*mmio_readb)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace); + uint16_t (*mmio_readw)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace); + uint32_t (*mmio_readl)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace); + uint64_t (*mmio_readq)(struct drm_i915_private *dev_priv, i915_reg_t r, bool trace); - void (*mmio_writeb)(struct drm_i915_private *dev_priv, off_t offset, + void (*mmio_writeb)(struct drm_i915_private *dev_priv, i915_reg_t r, uint8_t val, bool trace); - void (*mmio_writew)(struct drm_i915_private *dev_priv, off_t offset, + void (*mmio_writew)(struct drm_i915_private *dev_priv, i915_reg_t r, uint16_t val, bool trace); - void (*mmio_writel)(struct drm_i915_private *dev_priv, off_t offset, + void (*mmio_writel)(struct drm_i915_private *dev_priv, i915_reg_t r, uint32_t val, bool trace); - void (*mmio_writeq)(struct drm_i915_private *dev_priv, off_t offset, + void (*mmio_writeq)(struct drm_i915_private *dev_priv, i915_reg_t r, uint64_t val, bool trace); }; @@ -679,11 +716,11 @@ struct intel_uncore { enum forcewake_domain_id id; unsigned wake_count; struct timer_list timer; - u32 reg_set; + i915_reg_t reg_set; u32 val_set; u32 val_clear; - u32 reg_ack; - u32 reg_post; + i915_reg_t reg_ack; + i915_reg_t reg_post; u32 val_reset; } fw_domain[FW_DOMAIN_ID_COUNT]; }; @@ -693,25 +730,24 @@ struct intel_uncore { for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \ (i__) < FW_DOMAIN_ID_COUNT; \ (i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \ - if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__))) + for_each_if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__))) #define for_each_fw_domain(domain__, dev_priv__, i__) \ for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__) -enum csr_state { - FW_UNINITIALIZED = 0, - FW_LOADED, - FW_FAILED -}; +#define CSR_VERSION(major, minor) ((major) << 16 | (minor)) +#define CSR_VERSION_MAJOR(version) ((version) >> 16) +#define CSR_VERSION_MINOR(version) ((version) & 0xffff) struct intel_csr { + struct work_struct work; const char *fw_path; - __be32 *dmc_payload; + uint32_t *dmc_payload; uint32_t dmc_fw_size; + uint32_t version; uint32_t mmio_count; - uint32_t mmioaddr[8]; + i915_reg_t mmioaddr[8]; uint32_t mmiodata[8]; - enum csr_state state; }; #define DEV_INFO_FOR_EACH_FLAG(func, sep) \ @@ -729,6 +765,8 @@ struct intel_csr { func(is_valleyview) sep \ func(is_haswell) sep \ func(is_skylake) sep \ + func(is_broxton) sep \ + func(is_kabylake) sep \ func(is_preliminary) sep \ func(has_fbc) sep \ func(has_pipe_cxsr) sep \ @@ -805,11 +843,15 @@ struct i915_ctx_hang_stats { /* This must match up with the value previously used for execbuf2.rsvd1. */ #define DEFAULT_CONTEXT_HANDLE 0 + +#define CONTEXT_NO_ZEROMAP (1<<0) /** * struct intel_context - as the name implies, represents a context. * @ref: reference count. * @user_handle: userspace tracking identity for this context. * @remap_slice: l3 row remapping information. + * @flags: context specific flags: + * CONTEXT_NO_ZEROMAP: do not allow mapping things to page 0. * @file_priv: filp associated with this context (NULL for global default * context). * @hang_stats: information about the role of this context in possible GPU @@ -827,6 +869,7 @@ struct intel_context { int user_handle; uint8_t remap_slice; struct drm_i915_private *i915; + int flags; struct drm_i915_file_private *file_priv; struct i915_ctx_hang_stats hang_stats; struct i915_hw_ppgtt *ppgtt; @@ -838,7 +881,6 @@ struct intel_context { } legacy_hw_ctx; /* Execlists */ - bool rcs_initialized; struct { struct drm_i915_gem_object *state; struct intel_ringbuffer *ringbuf; @@ -853,10 +895,13 @@ enum fb_op_origin { ORIGIN_CPU, ORIGIN_CS, ORIGIN_FLIP, + ORIGIN_DIRTYFB, }; struct i915_fbc { - unsigned long uncompressed_size; + /* This is always the inner lock when overlapping with struct_mutex and + * it's the outer lock when overlapping with stolen_lock. */ + struct mutex lock; unsigned threshold; unsigned int fb_id; unsigned int possible_framebuffer_bits; @@ -869,29 +914,21 @@ struct i915_fbc { bool false_color; - /* Tracks whether the HW is actually enabled, not whether the feature is - * possible. */ bool enabled; + bool active; struct intel_fbc_work { - struct delayed_work work; - struct drm_crtc *crtc; + bool scheduled; + struct work_struct work; struct drm_framebuffer *fb; - } *fbc_work; - - enum no_fbc_reason { - FBC_OK, /* FBC is enabled */ - FBC_UNSUPPORTED, /* FBC is not supported by this chipset */ - FBC_NO_OUTPUT, /* no outputs enabled to compress */ - FBC_STOLEN_TOO_SMALL, /* not enough space for buffers */ - FBC_UNSUPPORTED_MODE, /* interlace or doublescanned mode */ - FBC_MODE_TOO_LARGE, /* mode too large for compression */ - FBC_BAD_PLANE, /* fbc not supported on plane */ - FBC_NOT_TILED, /* buffer not tiled */ - FBC_MULTIPLE_PIPES, /* more than one pipe active */ - FBC_MODULE_PARAM, - FBC_CHIP_DEFAULT, /* disabled by default on this chip */ - } no_fbc_reason; + unsigned long enable_jiffies; + } work; + + const char *no_fbc_reason; + + bool (*is_active)(struct drm_i915_private *dev_priv); + void (*activate)(struct intel_crtc *crtc); + void (*deactivate)(struct drm_i915_private *dev_priv); }; /** @@ -961,7 +998,7 @@ struct intel_gmbus { struct i2c_adapter adapter; u32 force_bit; u32 reg0; - u32 gpio_reg; + i915_reg_t gpio_reg; struct i2c_algo_bit_data bit_algo; struct drm_i915_private *dev_priv; }; @@ -980,7 +1017,7 @@ struct i915_suspend_saved_registers { u32 saveMI_ARB_STATE; u32 saveSWF0[16]; u32 saveSWF1[16]; - u32 saveSWF2[3]; + u32 saveSWF3[3]; uint64_t saveFENCE[I915_MAX_NUM_FENCES]; u32 savePCH_PORT_HOTPLUG; u16 saveGCDGMBUS; @@ -1082,7 +1119,6 @@ struct intel_gen6_power_mgmt { u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ u8 rp1_freq; /* "less than" RP0 power/freqency */ u8 rp0_freq; /* Non-overclocked max frequency. */ - u32 cz_freq; u8 up_threshold; /* Current %busy required to uplock */ u8 down_threshold; /* Current %busy required to downclock */ @@ -1201,6 +1237,10 @@ struct intel_l3_parity { struct i915_gem_mm { /** Memory allocator for GTT stolen memory */ struct drm_mm stolen; + /** Protects the usage of the GTT stolen memory allocator. This is + * always the inner lock when overlapping with struct_mutex. */ + struct mutex stolen_lock; + /** List of all objects in gtt_space. Used to restore gtt * mappings on resume */ struct list_head bound_list; @@ -1354,6 +1394,15 @@ enum modeset_restore { MODESET_SUSPENDED, }; +#define DP_AUX_A 0x40 +#define DP_AUX_B 0x10 +#define DP_AUX_C 0x20 +#define DP_AUX_D 0x30 + +#define DDC_PIN_B 0x05 +#define DDC_PIN_C 0x04 +#define DDC_PIN_D 0x06 + struct ddi_vbt_port_info { /* * This is an index in the HDMI/DVI DDI buffer translation table. @@ -1366,6 +1415,12 @@ struct ddi_vbt_port_info { uint8_t supports_dvi:1; uint8_t supports_hdmi:1; uint8_t supports_dp:1; + + uint8_t alternate_aux_channel; + uint8_t alternate_ddc_pin; + + uint8_t dp_boost_level; + uint8_t hdmi_boost_level; }; enum psr_lines_to_wait { @@ -1461,23 +1516,27 @@ struct ilk_wm_values { enum intel_ddb_partitioning partitioning; }; -struct vlv_wm_values { - struct { - uint16_t primary; - uint16_t sprite[2]; - uint8_t cursor; - } pipe[3]; +struct vlv_pipe_wm { + uint16_t primary; + uint16_t sprite[2]; + uint8_t cursor; +}; - struct { - uint16_t plane; - uint8_t cursor; - } sr; +struct vlv_sr_wm { + uint16_t plane; + uint8_t cursor; +}; +struct vlv_wm_values { + struct vlv_pipe_wm pipe[3]; + struct vlv_sr_wm sr; struct { uint8_t cursor; uint8_t sprite[2]; uint8_t primary; } ddl[3]; + uint8_t level; + bool cxsr; }; struct skl_ddb_entry { @@ -1501,8 +1560,7 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, struct skl_ddb_allocation { struct skl_ddb_entry pipe[I915_MAX_PIPES]; struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */ - struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* y-plane */ - struct skl_ddb_entry cursor[I915_MAX_PIPES]; + struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES]; }; struct skl_wm_values { @@ -1510,18 +1568,13 @@ struct skl_wm_values { struct skl_ddb_allocation ddb; uint32_t wm_linetime[I915_MAX_PIPES]; uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8]; - uint32_t cursor[I915_MAX_PIPES][8]; uint32_t plane_trans[I915_MAX_PIPES][I915_MAX_PLANES]; - uint32_t cursor_trans[I915_MAX_PIPES]; }; struct skl_wm_level { bool plane_en[I915_MAX_PLANES]; - bool cursor_en; uint16_t plane_res_b[I915_MAX_PLANES]; uint8_t plane_res_l[I915_MAX_PLANES]; - uint16_t cursor_res_b; - uint8_t cursor_res_l; }; /* @@ -1594,7 +1647,7 @@ struct i915_frontbuffer_tracking { }; struct i915_wa_reg { - u32 addr; + i915_reg_t addr; u32 value; /* bitmask representing WA bits */ u32 mask; @@ -1611,6 +1664,25 @@ struct i915_virtual_gpu { bool active; }; +struct i915_execbuffer_params { + struct drm_device *dev; + struct drm_file *file; + uint32_t dispatch_flags; + uint32_t args_batch_start_offset; + uint64_t batch_obj_vm_offset; + struct intel_engine_cs *ring; + struct drm_i915_gem_object *batch_obj; + struct intel_context *ctx; + struct drm_i915_gem_request *request; +}; + +/* used in computing the new watermarks state */ +struct intel_wm_config { + unsigned int num_pipes_active; + bool sprites_enabled; + bool sprites_scaled; +}; + struct drm_i915_private { struct drm_device *dev; struct kmem_cache *objects; @@ -1627,10 +1699,9 @@ struct drm_i915_private { struct i915_virtual_gpu vgpu; - struct intel_csr csr; + struct intel_guc guc; - /* Display CSR-related protection */ - struct mutex csr_lock; + struct intel_csr csr; struct intel_gmbus gmbus[GMBUS_NUM_PINS]; @@ -1646,6 +1717,8 @@ struct drm_i915_private { /* MMIO base address for MIPI regs */ uint32_t mipi_mmio_base; + uint32_t psr_mmio_base; + wait_queue_head_t gmbus_wait_queue; struct pci_dev *bridge_dev; @@ -1680,19 +1753,7 @@ struct drm_i915_private { u32 pm_rps_events; u32 pipestat_irq_mask[I915_MAX_PIPES]; - struct work_struct hotplug_work; - struct { - unsigned long hpd_last_jiffies; - int hpd_cnt; - enum { - HPD_ENABLED = 0, - HPD_DISABLED = 1, - HPD_MARK_DISABLED = 2 - } hpd_mark; - } hpd_stats[HPD_NUM_PINS]; - u32 hpd_event_bits; - struct delayed_work hotplug_reenable_work; - + struct i915_hotplug hotplug; struct i915_fbc fbc; struct i915_drrs drrs; struct intel_opregion opregion; @@ -1713,13 +1774,14 @@ struct drm_i915_private { struct mutex pps_mutex; struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */ - int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */ int num_fence_regs; /* 8 on pre-965, 16 otherwise */ unsigned int fsb_freq, mem_freq, is_ddr3; unsigned int skl_boot_cdclk; - unsigned int cdclk_freq; + unsigned int cdclk_freq, max_cdclk_freq; + unsigned int max_dotclk_freq; unsigned int hpll_freq; + unsigned int czclk_freq; /** * wq - Driver workqueue for GEM. @@ -1769,9 +1831,6 @@ struct drm_i915_private { /* Reclocking support */ bool render_reclock_avail; - bool lvds_downclock_avail; - /* indicates the reduced downclock for LVDS*/ - int lvds_downclock; struct i915_frontbuffer_tracking fb_tracking; @@ -1799,7 +1858,7 @@ struct drm_i915_private { struct drm_i915_gem_object *vlv_pctx; -#ifdef CONFIG_DRM_I915_FBDEV +#ifdef CONFIG_DRM_FBDEV_EMULATION /* list of fbdev register on this device */ struct intel_fbdev *fbdev; struct work_struct fbdev_suspend_work; @@ -1809,7 +1868,13 @@ struct drm_i915_private { struct drm_property *force_audio_property; /* hda/i915 audio component */ + struct i915_audio_component *audio_component; bool audio_component_registered; + /** + * av_mutex - mutex for audio/video sync + * + */ + struct mutex av_mutex; uint32_t hw_context_size; struct list_head context_list; @@ -1819,6 +1884,7 @@ struct drm_i915_private { u32 chv_phy_control; u32 suspend_count; + bool suspended_to_idle; struct i915_suspend_saved_registers regfile; struct vlv_s0ix_state vlv_s0ix_state; @@ -1841,6 +1907,9 @@ struct drm_i915_private { */ uint16_t skl_latency[8]; + /* Committed wm config */ + struct intel_wm_config config; + /* * The skl_wm_values structure is a bit too big for stack * allocation, so we keep the staging struct where we store @@ -1854,33 +1923,17 @@ struct drm_i915_private { struct skl_wm_values skl_hw; struct vlv_wm_values vlv; }; + + uint8_t max_level; } wm; struct i915_runtime_pm pm; - struct intel_digital_port *hpd_irq_port[I915_MAX_PORTS]; - u32 long_hpd_port_mask; - u32 short_hpd_port_mask; - struct work_struct dig_port_work; - - /* - * if we get a HPD irq from DP and a HPD irq from non-DP - * the non-DP HPD could block the workqueue on a mode config - * mutex getting, that userspace may have taken. However - * userspace is waiting on the DP workqueue to run which is - * blocked behind the non-DP one. - */ - struct workqueue_struct *dp_wq; - /* Abstract the submission mechanism (legacy ringbuffer or execlists) away */ struct { - int (*execbuf_submit)(struct drm_device *dev, struct drm_file *file, - struct intel_engine_cs *ring, - struct intel_context *ctx, + int (*execbuf_submit)(struct i915_execbuffer_params *params, struct drm_i915_gem_execbuffer2 *args, - struct list_head *vmas, - struct drm_i915_gem_object *batch_obj, - u64 exec_start, u32 flags); + struct list_head *vmas); int (*init_rings)(struct drm_device *dev); void (*cleanup_ring)(struct intel_engine_cs *ring); void (*stop_ring)(struct intel_engine_cs *ring); @@ -1888,6 +1941,9 @@ struct drm_i915_private { bool edp_low_vswing; + /* perform PHY state sanity checks? */ + bool chv_phy_assert[2]; + /* * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch * will be rejected. Instead look for a better place. @@ -1904,10 +1960,15 @@ static inline struct drm_i915_private *dev_to_i915(struct device *dev) return to_i915(dev_get_drvdata(dev)); } +static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc) +{ + return container_of(guc, struct drm_i915_private, guc); +} + /* Iterate over initialised rings */ #define for_each_ring(ring__, dev_priv__, i__) \ for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \ - if (((ring__) = &(dev_priv__)->ring[(i__)]), intel_ring_initialized((ring__))) + for_each_if ((((ring__) = &(dev_priv__)->ring[(i__)]), intel_ring_initialized((ring__)))) enum hdmi_force_audio { HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ @@ -1940,25 +2001,26 @@ struct drm_i915_gem_object_ops { /* * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is - * considered to be the frontbuffer for the given plane interface-vise. This + * considered to be the frontbuffer for the given plane interface-wise. This * doesn't mean that the hw necessarily already scans it out, but that any * rendering (by the cpu or gpu) will land in the frontbuffer eventually. * * We have one bit per pipe and per scanout plane type. */ -#define INTEL_FRONTBUFFER_BITS_PER_PIPE 4 +#define INTEL_MAX_SPRITE_BITS_PER_PIPE 5 +#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8 #define INTEL_FRONTBUFFER_BITS \ (INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES) #define INTEL_FRONTBUFFER_PRIMARY(pipe) \ (1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) #define INTEL_FRONTBUFFER_CURSOR(pipe) \ - (1 << (1 +(INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) -#define INTEL_FRONTBUFFER_SPRITE(pipe) \ - (1 << (2 +(INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) + (1 << (1 + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) +#define INTEL_FRONTBUFFER_SPRITE(pipe, plane) \ + (1 << (2 + plane + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) #define INTEL_FRONTBUFFER_OVERLAY(pipe) \ - (1 << (3 +(INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) + (1 << (2 + INTEL_MAX_SPRITE_BITS_PER_PIPE + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))) #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ - (0xf << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) + (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) struct drm_i915_gem_object { struct drm_gem_object base; @@ -2148,7 +2210,8 @@ struct drm_i915_gem_request { struct intel_context *ctx; struct intel_ringbuffer *ringbuf; - /** Batch buffer related to this request if any */ + /** Batch buffer related to this request if any (used for + error state dump only) */ struct drm_i915_gem_object *batch_obj; /** Time at which this request was emitted, in jiffies. */ @@ -2186,8 +2249,12 @@ struct drm_i915_gem_request { }; int i915_gem_request_alloc(struct intel_engine_cs *ring, - struct intel_context *ctx); + struct intel_context *ctx, + struct drm_i915_gem_request **req_out); +void i915_gem_request_cancel(struct drm_i915_gem_request *req); void i915_gem_request_free(struct kref *req_ref); +int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, + struct drm_file *file); static inline uint32_t i915_gem_request_get_seqno(struct drm_i915_gem_request *req) @@ -2357,6 +2424,15 @@ struct drm_i915_cmd_table { #define INTEL_DEVID(p) (INTEL_INFO(p)->device_id) #define INTEL_REVID(p) (__I915__(p)->dev->pdev->revision) +#define REVID_FOREVER 0xff +/* + * Return true if revision is in range [since,until] inclusive. + * + * Use 0 for open-ended since, and REVID_FOREVER for open-ended until. + */ +#define IS_REVID(p, since, until) \ + (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) + #define IS_I830(dev) (INTEL_DEVID(dev) == 0x3577) #define IS_845G(dev) (INTEL_DEVID(dev) == 0x2562) #define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) @@ -2383,7 +2459,8 @@ struct drm_i915_cmd_table { #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell) #define IS_BROADWELL(dev) (!INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev)) #define IS_SKYLAKE(dev) (INTEL_INFO(dev)->is_skylake) -#define IS_BROXTON(dev) (!INTEL_INFO(dev)->is_skylake && IS_GEN9(dev)) +#define IS_BROXTON(dev) (INTEL_INFO(dev)->is_broxton) +#define IS_KABYLAKE(dev) (INTEL_INFO(dev)->is_kabylake) #define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) #define IS_HSW_EARLY_SDV(dev) (IS_HASWELL(dev) && \ (INTEL_DEVID(dev) & 0xFF00) == 0x0C00) @@ -2391,6 +2468,9 @@ struct drm_i915_cmd_table { ((INTEL_DEVID(dev) & 0xf) == 0x6 || \ (INTEL_DEVID(dev) & 0xf) == 0xb || \ (INTEL_DEVID(dev) & 0xf) == 0xe)) +/* ULX machines are also considered ULT. */ +#define IS_BDW_ULX(dev) (IS_BROADWELL(dev) && \ + (INTEL_DEVID(dev) & 0xf) == 0xe) #define IS_BDW_GT3(dev) (IS_BROADWELL(dev) && \ (INTEL_DEVID(dev) & 0x00F0) == 0x0020) #define IS_HSW_ULT(dev) (IS_HASWELL(dev) && \ @@ -2400,18 +2480,36 @@ struct drm_i915_cmd_table { /* ULX machines are also considered ULT. */ #define IS_HSW_ULX(dev) (INTEL_DEVID(dev) == 0x0A0E || \ INTEL_DEVID(dev) == 0x0A1E) +#define IS_SKL_ULT(dev) (INTEL_DEVID(dev) == 0x1906 || \ + INTEL_DEVID(dev) == 0x1913 || \ + INTEL_DEVID(dev) == 0x1916 || \ + INTEL_DEVID(dev) == 0x1921 || \ + INTEL_DEVID(dev) == 0x1926) +#define IS_SKL_ULX(dev) (INTEL_DEVID(dev) == 0x190E || \ + INTEL_DEVID(dev) == 0x1915 || \ + INTEL_DEVID(dev) == 0x191E) +#define IS_SKL_GT3(dev) (IS_SKYLAKE(dev) && \ + (INTEL_DEVID(dev) & 0x00F0) == 0x0020) +#define IS_SKL_GT4(dev) (IS_SKYLAKE(dev) && \ + (INTEL_DEVID(dev) & 0x00F0) == 0x0030) + #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary) -#define SKL_REVID_A0 (0x0) -#define SKL_REVID_B0 (0x1) -#define SKL_REVID_C0 (0x2) -#define SKL_REVID_D0 (0x3) -#define SKL_REVID_E0 (0x4) -#define SKL_REVID_F0 (0x5) +#define SKL_REVID_A0 0x0 +#define SKL_REVID_B0 0x1 +#define SKL_REVID_C0 0x2 +#define SKL_REVID_D0 0x3 +#define SKL_REVID_E0 0x4 +#define SKL_REVID_F0 0x5 -#define BXT_REVID_A0 (0x0) -#define BXT_REVID_B0 (0x3) -#define BXT_REVID_C0 (0x6) +#define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until)) + +#define BXT_REVID_A0 0x0 +#define BXT_REVID_A1 0x1 +#define BXT_REVID_B0 0x3 +#define BXT_REVID_C0 0x9 + +#define IS_BXT_REVID(p, since, until) (IS_BROXTON(p) && IS_REVID(p, since, until)) /* * The genX designation typically refers to the render engine, so render @@ -2445,7 +2543,8 @@ struct drm_i915_cmd_table { #define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6) #define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 8) #define USES_PPGTT(dev) (i915.enable_ppgtt) -#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt == 2) +#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt >= 2) +#define USES_FULL_48BIT_PPGTT(dev) (i915.enable_ppgtt == 3) #define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) #define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) @@ -2466,9 +2565,6 @@ struct drm_i915_cmd_table { */ #define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \ IS_I915GM(dev))) -#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) -#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev)) -#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev)) #define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) #define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) @@ -2485,14 +2581,23 @@ struct drm_i915_cmd_table { #define HAS_FPGA_DBG_UNCLAIMED(dev) (INTEL_INFO(dev)->has_fpga_dbg) #define HAS_PSR(dev) (IS_HASWELL(dev) || IS_BROADWELL(dev) || \ IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev) || \ - IS_SKYLAKE(dev)) + IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) #define HAS_RUNTIME_PM(dev) (IS_GEN6(dev) || IS_HASWELL(dev) || \ IS_BROADWELL(dev) || IS_VALLEYVIEW(dev) || \ - IS_SKYLAKE(dev)) + IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) #define HAS_RC6(dev) (INTEL_INFO(dev)->gen >= 6) #define HAS_RC6p(dev) (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev)) -#define HAS_CSR(dev) (IS_SKYLAKE(dev)) +#define HAS_CSR(dev) (IS_GEN9(dev)) + +#define HAS_GUC_UCODE(dev) (IS_GEN9(dev)) +#define HAS_GUC_SCHED(dev) (IS_GEN9(dev)) + +#define HAS_RESOURCE_STREAMER(dev) (IS_HASWELL(dev) || \ + INTEL_INFO(dev)->gen >= 8) + +#define HAS_CORE_RING_FREQ(dev) (INTEL_INFO(dev)->gen >= 6 && \ + !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev)) #define INTEL_PCH_DEVICE_ID_MASK 0xff00 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 @@ -2502,10 +2607,14 @@ struct drm_i915_cmd_table { #define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE 0x9c00 #define INTEL_PCH_SPT_DEVICE_ID_TYPE 0xA100 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE 0x9D00 +#define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 +#define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ #define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type) #define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT) #define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT) +#define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) +#define HAS_PCH_LPT_H(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) #define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT) #define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX) #define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP) @@ -2525,19 +2634,19 @@ struct drm_i915_cmd_table { extern const struct drm_ioctl_desc i915_ioctls[]; extern int i915_max_ioctl; -extern int i915_suspend_legacy(struct drm_device *dev, pm_message_t state); -extern int i915_resume_legacy(struct drm_device *dev); +extern int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state); +extern int i915_resume_switcheroo(struct drm_device *dev); /* i915_params.c */ struct i915_params { int modeset; int panel_ignore_lid; int semaphores; - unsigned int lvds_downclock; int lvds_channel_mode; int panel_use_ssc; int vbt_sdvo_panel_type; int enable_rc6; + int enable_dc; int enable_fbc; int enable_ppgtt; int enable_execlists; @@ -2555,6 +2664,8 @@ struct i915_params { bool reset; bool disable_display; bool disable_vtd_wa; + bool enable_guc_submission; + int guc_log_level; int use_mmio_flip; int mmio_debug; bool verbose_state_checks; @@ -2572,20 +2683,25 @@ extern void i915_driver_preclose(struct drm_device *dev, struct drm_file *file); extern void i915_driver_postclose(struct drm_device *dev, struct drm_file *file); -extern int i915_driver_device_is_agp(struct drm_device * dev); #ifdef CONFIG_COMPAT extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); #endif extern int intel_gpu_reset(struct drm_device *dev); +extern bool intel_has_gpu_reset(struct drm_device *dev); extern int i915_reset(struct drm_device *dev); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); + +/* intel_hotplug.c */ +void intel_hpd_irq_handler(struct drm_device *dev, u32 pin_mask, u32 long_mask); +void intel_hpd_init(struct drm_i915_private *dev_priv); +void intel_hpd_init_work(struct drm_i915_private *dev_priv); void intel_hpd_cancel_work(struct drm_i915_private *dev_priv); -void i915_firmware_load_error_print(const char *fw_path, int err); +bool intel_hpd_pin_to_port(enum hpd_pin pin, enum port *port); /* i915_irq.c */ void i915_queue_hangcheck(struct drm_device *dev); @@ -2594,7 +2710,6 @@ void i915_handle_error(struct drm_device *dev, bool wedged, const char *fmt, ...); extern void intel_irq_init(struct drm_i915_private *dev_priv); -extern void intel_hpd_init(struct drm_i915_private *dev_priv); int intel_irq_install(struct drm_i915_private *dev_priv); void intel_irq_uninstall(struct drm_i915_private *dev_priv); @@ -2633,17 +2748,50 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); -void -ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask); -void -ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask); +void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, + uint32_t mask, + uint32_t bits); +void ilk_update_display_irq(struct drm_i915_private *dev_priv, + uint32_t interrupt_mask, + uint32_t enabled_irq_mask); +static inline void +ilk_enable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) +{ + ilk_update_display_irq(dev_priv, bits, bits); +} +static inline void +ilk_disable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits) +{ + ilk_update_display_irq(dev_priv, bits, 0); +} +void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, + enum pipe pipe, + uint32_t interrupt_mask, + uint32_t enabled_irq_mask); +static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv, + enum pipe pipe, uint32_t bits) +{ + bdw_update_pipe_irq(dev_priv, pipe, bits, bits); +} +static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv, + enum pipe pipe, uint32_t bits) +{ + bdw_update_pipe_irq(dev_priv, pipe, bits, 0); +} void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, uint32_t interrupt_mask, uint32_t enabled_irq_mask); -#define ibx_enable_display_interrupt(dev_priv, bits) \ - ibx_display_interrupt_update((dev_priv), (bits), (bits)) -#define ibx_disable_display_interrupt(dev_priv, bits) \ - ibx_display_interrupt_update((dev_priv), (bits), 0) +static inline void +ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) +{ + ibx_display_interrupt_update(dev_priv, bits, bits); +} +static inline void +ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) +{ + ibx_display_interrupt_update(dev_priv, bits, 0); +} + /* i915_gem.c */ int i915_gem_create_ioctl(struct drm_device *dev, void *data, @@ -2661,19 +2809,11 @@ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void i915_gem_execbuffer_move_to_active(struct list_head *vmas, - struct intel_engine_cs *ring); -void i915_gem_execbuffer_retire_commands(struct drm_device *dev, - struct drm_file *file, - struct intel_engine_cs *ring, - struct drm_i915_gem_object *obj); -int i915_gem_ringbuffer_submission(struct drm_device *dev, - struct drm_file *file, - struct intel_engine_cs *ring, - struct intel_context *ctx, + struct drm_i915_gem_request *req); +void i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params); +int i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params, struct drm_i915_gem_execbuffer2 *args, - struct list_head *vmas, - struct drm_i915_gem_object *batch_obj, - u64 exec_start, u32 flags); + struct list_head *vmas); int i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_execbuffer2(struct drm_device *dev, void *data, @@ -2706,8 +2846,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops); struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, size_t size); -void i915_init_vm(struct drm_i915_private *dev_priv, - struct i915_address_space *vm); +struct drm_i915_gem_object *i915_gem_object_create_from_data( + struct drm_device *dev, const void *data, size_t size); void i915_gem_free_object(struct drm_gem_object *obj); void i915_gem_vma_destroy(struct i915_vma *vma); @@ -2718,6 +2858,8 @@ void i915_gem_vma_destroy(struct i915_vma *vma); #define PIN_OFFSET_BIAS (1<<3) #define PIN_USER (1<<4) #define PIN_UPDATE (1<<5) +#define PIN_ZONE_4G (1<<6) +#define PIN_HIGH (1<<7) #define PIN_OFFSET_MASK (~4095) int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj, @@ -2733,6 +2875,11 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags); int __must_check i915_vma_unbind(struct i915_vma *vma); +/* + * BEWARE: Do not use the function below unless you can _absolutely_ + * _guarantee_ VMA in question is _not in use_ anywhere. + */ +int __must_check __i915_vma_unbind_no_wait(struct i915_vma *vma); int i915_gem_object_put_pages(struct drm_i915_gem_object *obj); void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); @@ -2780,9 +2927,10 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); int i915_gem_object_sync(struct drm_i915_gem_object *obj, - struct intel_engine_cs *to); + struct intel_engine_cs *to, + struct drm_i915_gem_request **to_req); void i915_vma_move_to_active(struct i915_vma *vma, - struct intel_engine_cs *ring); + struct drm_i915_gem_request *req); int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); @@ -2811,11 +2959,6 @@ static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req, int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno); int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); -int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj); -int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj); - -bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj); -void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj); struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *ring); @@ -2824,7 +2967,6 @@ bool i915_gem_retire_requests(struct drm_device *dev); void i915_gem_retire_requests_ring(struct intel_engine_cs *ring); int __must_check i915_gem_check_wedge(struct i915_gpu_error *error, bool interruptible); -int __must_check i915_gem_check_olr(struct drm_i915_gem_request *req); static inline bool i915_reset_in_progress(struct i915_gpu_error *error) { @@ -2859,16 +3001,18 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); int __must_check i915_gem_init(struct drm_device *dev); int i915_gem_init_rings(struct drm_device *dev); int __must_check i915_gem_init_hw(struct drm_device *dev); -int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice); +int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice); void i915_gem_init_swizzling(struct drm_device *dev); void i915_gem_cleanup_ringbuffer(struct drm_device *dev); int __must_check i915_gpu_idle(struct drm_device *dev); int __must_check i915_gem_suspend(struct drm_device *dev); -int __i915_add_request(struct intel_engine_cs *ring, - struct drm_file *file, - struct drm_i915_gem_object *batch_obj); -#define i915_add_request(ring) \ - __i915_add_request(ring, NULL, NULL) +void __i915_add_request(struct drm_i915_gem_request *req, + struct drm_i915_gem_object *batch_obj, + bool flush_caches); +#define i915_add_request(req) \ + __i915_add_request(req, NULL, true) +#define i915_add_request_no_flush(req) \ + __i915_add_request(req, NULL, false) int __i915_wait_request(struct drm_i915_gem_request *req, unsigned reset_counter, bool interruptible, @@ -2887,7 +3031,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); int __must_check i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, - struct intel_engine_cs *pipelined, const struct i915_ggtt_view *view); void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, const struct i915_ggtt_view *view); @@ -2911,15 +3054,11 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, struct dma_buf *i915_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gem_obj, int flags); -void i915_gem_restore_fences(struct drm_device *dev); - -unsigned long -i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, - const struct i915_ggtt_view *view); -unsigned long -i915_gem_obj_offset(struct drm_i915_gem_object *o, - struct i915_address_space *vm); -static inline unsigned long +u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, + const struct i915_ggtt_view *view); +u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, + struct i915_address_space *vm); +static inline u64 i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *o) { return i915_gem_obj_ggtt_offset_view(o, &i915_ggtt_view_normal); @@ -3007,15 +3146,27 @@ i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj) i915_gem_object_ggtt_unpin_view(obj, &i915_ggtt_view_normal); } +/* i915_gem_fence.c */ +int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj); +int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj); + +bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj); +void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj); + +void i915_gem_restore_fences(struct drm_device *dev); + +void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); +void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); +void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); + /* i915_gem_context.c */ int __must_check i915_gem_context_init(struct drm_device *dev); void i915_gem_context_fini(struct drm_device *dev); void i915_gem_context_reset(struct drm_device *dev); int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); -int i915_gem_context_enable(struct drm_i915_private *dev_priv); +int i915_gem_context_enable(struct drm_i915_gem_request *req); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); -int i915_switch_context(struct intel_engine_cs *ring, - struct intel_context *to); +int i915_switch_context(struct drm_i915_gem_request *req); struct intel_context * i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id); void i915_gem_context_free(struct kref *ctx_ref); @@ -3055,7 +3206,6 @@ int __must_check i915_gem_evict_something(struct drm_device *dev, unsigned long end, unsigned flags); int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle); -int i915_gem_evict_everything(struct drm_device *dev); /* belongs in i915_gem_gtt.h */ static inline void i915_gem_chipset_flush(struct drm_device *dev) @@ -3065,9 +3215,16 @@ static inline void i915_gem_chipset_flush(struct drm_device *dev) } /* i915_gem_stolen.c */ +int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv, + struct drm_mm_node *node, u64 size, + unsigned alignment); +int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, + struct drm_mm_node *node, u64 size, + unsigned alignment, u64 start, + u64 end); +void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, + struct drm_mm_node *node); int i915_gem_init_stolen(struct drm_device *dev); -int i915_gem_stolen_setup_compression(struct drm_device *dev, int size, int fb_cpp); -void i915_gem_stolen_cleanup_compression(struct drm_device *dev); void i915_gem_cleanup_stolen(struct drm_device *dev); struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_device *dev, u32 size); @@ -3079,11 +3236,12 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, /* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, - long target, + unsigned long target, unsigned flags); #define I915_SHRINK_PURGEABLE 0x1 #define I915_SHRINK_UNBOUND 0x2 #define I915_SHRINK_BOUND 0x4 +#define I915_SHRINK_ACTIVE 0x8 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); void i915_gem_shrinker_init(struct drm_i915_private *dev_priv); @@ -3097,10 +3255,6 @@ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_objec obj->tiling_mode != I915_TILING_NONE; } -void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); - /* i915_gem_debug.c */ #if WATCH_LISTS int i915_verify_lists(struct drm_device *dev); @@ -3222,8 +3376,7 @@ extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); extern void intel_connector_unregister(struct intel_connector *); extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); -extern void intel_modeset_setup_hw_state(struct drm_device *dev, - bool force_restore); +extern void intel_display_resume(struct drm_device *dev); extern void i915_redisable_vga(struct drm_device *dev); extern void i915_redisable_vga_power_on(struct drm_device *dev); extern bool ironlake_set_drps(struct drm_device *dev, u8 val); @@ -3232,7 +3385,6 @@ extern void intel_set_rps(struct drm_device *dev, u8 val); extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); extern void intel_detect_pch(struct drm_device *dev); -extern int intel_trans_dp_port_sel(struct drm_crtc *crtc); extern int intel_enable_rc6(const struct drm_device *dev); extern bool i915_semaphore_is_enabled(struct drm_device *dev); @@ -3303,18 +3455,44 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); #define I915_READ64(reg) dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true) #define I915_READ64_2x32(lower_reg, upper_reg) ({ \ - u32 upper, lower, tmp; \ - tmp = I915_READ(upper_reg); \ + u32 upper, lower, old_upper, loop = 0; \ + upper = I915_READ(upper_reg); \ do { \ - upper = tmp; \ + old_upper = upper; \ lower = I915_READ(lower_reg); \ - tmp = I915_READ(upper_reg); \ - } while (upper != tmp); \ + upper = I915_READ(upper_reg); \ + } while (upper != old_upper && loop++ < 2); \ (u64)upper << 32 | lower; }) #define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) #define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) +#define __raw_read(x, s) \ +static inline uint##x##_t __raw_i915_read##x(struct drm_i915_private *dev_priv, \ + i915_reg_t reg) \ +{ \ + return read##s(dev_priv->regs + i915_mmio_reg_offset(reg)); \ +} + +#define __raw_write(x, s) \ +static inline void __raw_i915_write##x(struct drm_i915_private *dev_priv, \ + i915_reg_t reg, uint##x##_t val) \ +{ \ + write##s(val, dev_priv->regs + i915_mmio_reg_offset(reg)); \ +} +__raw_read(8, b) +__raw_read(16, w) +__raw_read(32, l) +__raw_read(64, q) + +__raw_write(8, b) +__raw_write(16, w) +__raw_write(32, l) +__raw_write(64, q) + +#undef __raw_read +#undef __raw_write + /* These are untraced mmio-accessors that are only valid to be used inside * criticial sections inside IRQ handlers where forcewake is explicitly * controlled. @@ -3322,8 +3500,8 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); * Note: Should only be used between intel_uncore_forcewake_irqlock() and * intel_uncore_forcewake_irqunlock(). */ -#define I915_READ_FW(reg__) readl(dev_priv->regs + (reg__)) -#define I915_WRITE_FW(reg__, val__) writel(val__, dev_priv->regs + (reg__)) +#define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) +#define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) #define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__) /* "Broadcast RGB" property */ @@ -3331,7 +3509,7 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); #define INTEL_BROADCAST_RGB_FULL 1 #define INTEL_BROADCAST_RGB_LIMITED 2 -static inline uint32_t i915_vgacntrl_reg(struct drm_device *dev) +static inline i915_reg_t i915_vgacntrl_reg(struct drm_device *dev) { if (IS_VALLEYVIEW(dev)) return VLV_VGACNTRL; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 52b446b27b4d..b7d7cecdddf6 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -46,11 +46,6 @@ static void i915_gem_object_retire__write(struct drm_i915_gem_object *obj); static void i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring); -static void i915_gem_write_fence(struct drm_device *dev, int reg, - struct drm_i915_gem_object *obj); -static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, - struct drm_i915_fence_reg *fence, - bool enable); static bool cpu_cache_is_coherent(struct drm_device *dev, enum i915_cache_level level) @@ -66,18 +61,6 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) return obj->pin_display; } -static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) -{ - if (obj->tiling_mode) - i915_gem_release_mmap(obj); - - /* As we do not have an associated fence register, we will force - * a tiling change if we ever need to acquire one. - */ - obj->fence_dirty = false; - obj->fence_reg = I915_FENCE_REG_NONE; -} - /* some bookkeeping */ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) @@ -149,14 +132,18 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_get_aperture *args = data; - struct drm_i915_gem_object *obj; + struct i915_gtt *ggtt = &dev_priv->gtt; + struct i915_vma *vma; size_t pinned; pinned = 0; mutex_lock(&dev->struct_mutex); - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) - if (i915_gem_obj_is_pinned(obj)) - pinned += i915_gem_obj_ggtt_size(obj); + list_for_each_entry(vma, &ggtt->base.active_list, mm_list) + if (vma->pin_count) + pinned += vma->node.size; + list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list) + if (vma->pin_count) + pinned += vma->node.size; mutex_unlock(&dev->struct_mutex); args->aper_size = dev_priv->gtt.base.total; @@ -347,7 +334,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, if (ret) return ret; - intel_fb_obj_invalidate(obj, NULL, ORIGIN_CPU); + intel_fb_obj_invalidate(obj, ORIGIN_CPU); if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { unsigned long unwritten; @@ -368,7 +355,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, i915_gem_chipset_flush(dev); out: - intel_fb_obj_flush(obj, false); + intel_fb_obj_flush(obj, false, ORIGIN_CPU); return ret; } @@ -801,7 +788,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, offset = i915_gem_obj_ggtt_offset(obj) + args->offset; - intel_fb_obj_invalidate(obj, NULL, ORIGIN_GTT); + intel_fb_obj_invalidate(obj, ORIGIN_GTT); while (remain > 0) { /* Operation in this page @@ -832,7 +819,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, } out_flush: - intel_fb_obj_flush(obj, false); + intel_fb_obj_flush(obj, false, ORIGIN_GTT); out_unpin: i915_gem_object_ggtt_unpin(obj); out: @@ -945,7 +932,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev, if (ret) return ret; - intel_fb_obj_invalidate(obj, NULL, ORIGIN_CPU); + intel_fb_obj_invalidate(obj, ORIGIN_CPU); i915_gem_object_pin_pages(obj); @@ -1018,14 +1005,16 @@ out: if (!needs_clflush_after && obj->base.write_domain != I915_GEM_DOMAIN_CPU) { if (i915_gem_clflush_object(obj, obj->pin_display)) - i915_gem_chipset_flush(dev); + needs_clflush_after = true; } } if (needs_clflush_after) i915_gem_chipset_flush(dev); + else + obj->cache_dirty = true; - intel_fb_obj_flush(obj, false); + intel_fb_obj_flush(obj, false, ORIGIN_CPU); return ret; } @@ -1146,23 +1135,6 @@ i915_gem_check_wedge(struct i915_gpu_error *error, return 0; } -/* - * Compare arbitrary request against outstanding lazy request. Emit on match. - */ -int -i915_gem_check_olr(struct drm_i915_gem_request *req) -{ - int ret; - - WARN_ON(!mutex_is_locked(&req->ring->dev->struct_mutex)); - - ret = 0; - if (req == req->ring->outstanding_lazy_request) - ret = i915_add_request(req->ring); - - return ret; -} - static void fake_irq(unsigned long data) { wake_up_process((struct task_struct *)data); @@ -1238,8 +1210,16 @@ int __i915_wait_request(struct drm_i915_gem_request *req, if (i915_gem_request_completed(req, true)) return 0; - timeout_expire = timeout ? - jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0; + timeout_expire = 0; + if (timeout) { + if (WARN_ON(*timeout < 0)) + return -EINVAL; + + if (*timeout == 0) + return -ETIME; + + timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout); + } if (INTEL_INFO(dev_priv)->gen >= 6) gen6_rps_boost(dev_priv, rps, req->emitted_jiffies); @@ -1334,6 +1314,33 @@ out: return ret; } +int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, + struct drm_file *file) +{ + struct drm_i915_private *dev_private; + struct drm_i915_file_private *file_priv; + + WARN_ON(!req || !file || req->file_priv); + + if (!req || !file) + return -EINVAL; + + if (req->file_priv) + return -EINVAL; + + dev_private = req->ring->dev->dev_private; + file_priv = file->driver_priv; + + spin_lock(&file_priv->mm.lock); + req->file_priv = file_priv; + list_add_tail(&req->client_list, &file_priv->mm.request_list); + spin_unlock(&file_priv->mm.lock); + + req->pid = get_pid(task_pid(current)); + + return 0; +} + static inline void i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) { @@ -1346,6 +1353,9 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) list_del(&request->client_list); request->file_priv = NULL; spin_unlock(&file_priv->mm.lock); + + put_pid(request->pid); + request->pid = NULL; } static void i915_gem_request_retire(struct drm_i915_gem_request *request) @@ -1365,8 +1375,6 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) list_del_init(&request->list); i915_gem_request_remove_from_client(request); - put_pid(request->pid); - i915_gem_request_unreference(request); } @@ -1415,10 +1423,6 @@ i915_wait_request(struct drm_i915_gem_request *req) if (ret) return ret; - ret = i915_gem_check_olr(req); - if (ret) - return ret; - ret = __i915_wait_request(req, atomic_read(&dev_priv->gpu_error.reset_counter), interruptible, NULL, NULL); @@ -1518,10 +1522,6 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, if (req == NULL) return 0; - ret = i915_gem_check_olr(req); - if (ret) - goto err; - requests[n++] = i915_gem_request_reference(req); } else { for (i = 0; i < I915_NUM_RINGS; i++) { @@ -1531,10 +1531,6 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, if (req == NULL) continue; - ret = i915_gem_check_olr(req); - if (ret) - goto err; - requests[n++] = i915_gem_request_reference(req); } } @@ -1545,7 +1541,6 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, NULL, rps); mutex_lock(&dev->struct_mutex); -err: for (i = 0; i < n; i++) { if (ret == 0) i915_gem_object_retire_request(obj, requests[i]); @@ -1613,6 +1608,11 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, else ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); + if (write_domain != 0) + intel_fb_obj_invalidate(obj, + write_domain == I915_GEM_DOMAIN_GTT ? + ORIGIN_GTT : ORIGIN_CPU); + unref: drm_gem_object_unreference(&obj->base); unlock: @@ -1721,8 +1721,8 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, /** * i915_gem_fault - fault a page into the GTT - * vma: VMA in question - * vmf: fault info + * @vma: VMA in question + * @vmf: fault info * * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped * from userspace. The fault handler takes care of binding the object to @@ -2224,9 +2224,8 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) * Fail silently without starting the shrinker */ mapping = file_inode(obj->base.filp)->i_mapping; - gfp = mapping_gfp_mask(mapping); - gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD; - gfp &= ~(__GFP_IO | __GFP_WAIT); + gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM)); + gfp |= __GFP_NORETRY | __GFP_NOWARN; sg = st->sgl; st->nents = 0; for (i = 0; i < page_count; i++) { @@ -2349,9 +2348,12 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj) } void i915_vma_move_to_active(struct i915_vma *vma, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { struct drm_i915_gem_object *obj = vma->obj; + struct intel_engine_cs *ring; + + ring = i915_gem_request_get_ring(req); /* Add a reference if we're newly entering the active list. */ if (obj->active == 0) @@ -2359,8 +2361,7 @@ void i915_vma_move_to_active(struct i915_vma *vma, obj->active |= intel_ring_flag(ring); list_move_tail(&obj->ring_list[ring->id], &ring->active_list); - i915_gem_request_assign(&obj->last_read_req[ring->id], - intel_ring_get_request(ring)); + i915_gem_request_assign(&obj->last_read_req[ring->id], req); list_move_tail(&vma->mm_list, &vma->vm->active_list); } @@ -2372,7 +2373,7 @@ i915_gem_object_retire__write(struct drm_i915_gem_object *obj) RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring))); i915_gem_request_assign(&obj->last_write_req, NULL); - intel_fb_obj_flush(obj, true); + intel_fb_obj_flush(obj, true, ORIGIN_CS); } static void @@ -2393,6 +2394,13 @@ i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring) if (obj->active) return; + /* Bump our place on the bound list to keep it roughly in LRU order + * so that we don't steal from recently used but inactive objects + * (unless we are forced to ofc!) + */ + list_move_tail(&obj->global_list, + &to_i915(obj->base.dev)->mm.bound_list); + list_for_each_entry(vma, &obj->vma_list, vma_link) { if (!list_empty(&vma->mm_list)) list_move_tail(&vma->mm_list, &vma->vm->inactive_list); @@ -2472,24 +2480,34 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) return 0; } -int __i915_add_request(struct intel_engine_cs *ring, - struct drm_file *file, - struct drm_i915_gem_object *obj) +/* + * NB: This function is not allowed to fail. Doing so would mean the the + * request is not being tracked for completion but the work itself is + * going to happen on the hardware. This would be a Bad Thing(tm). + */ +void __i915_add_request(struct drm_i915_gem_request *request, + struct drm_i915_gem_object *obj, + bool flush_caches) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - struct drm_i915_gem_request *request; + struct intel_engine_cs *ring; + struct drm_i915_private *dev_priv; struct intel_ringbuffer *ringbuf; u32 request_start; int ret; - request = ring->outstanding_lazy_request; if (WARN_ON(request == NULL)) - return -ENOMEM; + return; - if (i915.enable_execlists) { - ringbuf = request->ctx->engine[ring->id].ringbuf; - } else - ringbuf = ring->buffer; + ring = request->ring; + dev_priv = ring->dev->dev_private; + ringbuf = request->ringbuf; + + /* + * To ensure that this call will not fail, space for its emissions + * should already have been reserved in the ring buffer. Let the ring + * know that it is time to use that space up. + */ + intel_ring_reserved_space_use(ringbuf); request_start = intel_ring_get_tail(ringbuf); /* @@ -2499,14 +2517,13 @@ int __i915_add_request(struct intel_engine_cs *ring, * is that the flush _must_ happen before the next request, no matter * what. */ - if (i915.enable_execlists) { - ret = logical_ring_flush_all_caches(ringbuf, request->ctx); - if (ret) - return ret; - } else { - ret = intel_ring_flush_all_caches(ring); - if (ret) - return ret; + if (flush_caches) { + if (i915.enable_execlists) + ret = logical_ring_flush_all_caches(request); + else + ret = intel_ring_flush_all_caches(request); + /* Not allowed to fail! */ + WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret); } /* Record the position of the start of the request so that @@ -2516,17 +2533,15 @@ int __i915_add_request(struct intel_engine_cs *ring, */ request->postfix = intel_ring_get_tail(ringbuf); - if (i915.enable_execlists) { - ret = ring->emit_request(ringbuf, request); - if (ret) - return ret; - } else { - ret = ring->add_request(ring); - if (ret) - return ret; + if (i915.enable_execlists) + ret = ring->emit_request(request); + else { + ret = ring->add_request(request); request->tail = intel_ring_get_tail(ringbuf); } + /* Not allowed to fail! */ + WARN(ret, "emit|add_request failed: %d!\n", ret); request->head = request_start; @@ -2538,34 +2553,11 @@ int __i915_add_request(struct intel_engine_cs *ring, */ request->batch_obj = obj; - if (!i915.enable_execlists) { - /* Hold a reference to the current context so that we can inspect - * it later in case a hangcheck error event fires. - */ - request->ctx = ring->last_context; - if (request->ctx) - i915_gem_context_reference(request->ctx); - } - request->emitted_jiffies = jiffies; ring->last_submitted_seqno = request->seqno; list_add_tail(&request->list, &ring->request_list); - request->file_priv = NULL; - - if (file) { - struct drm_i915_file_private *file_priv = file->driver_priv; - - spin_lock(&file_priv->mm.lock); - request->file_priv = file_priv; - list_add_tail(&request->client_list, - &file_priv->mm.request_list); - spin_unlock(&file_priv->mm.lock); - - request->pid = get_pid(task_pid(current)); - } trace_i915_gem_request_add(request); - ring->outstanding_lazy_request = NULL; i915_queue_hangcheck(ring->dev); @@ -2574,7 +2566,8 @@ int __i915_add_request(struct intel_engine_cs *ring, round_jiffies_up_relative(HZ)); intel_mark_busy(dev_priv->dev); - return 0; + /* Sanity check that the reserved size was large enough. */ + intel_ring_reserved_space_end(ringbuf); } static bool i915_context_is_banned(struct drm_i915_private *dev_priv, @@ -2628,12 +2621,13 @@ void i915_gem_request_free(struct kref *req_ref) typeof(*req), ref); struct intel_context *ctx = req->ctx; + if (req->file_priv) + i915_gem_request_remove_from_client(req); + if (ctx) { if (i915.enable_execlists) { - struct intel_engine_cs *ring = req->ring; - - if (ctx != ring->default_context) - intel_lr_context_unpin(ring, ctx); + if (ctx != req->ring->default_context) + intel_lr_context_unpin(req); } i915_gem_context_unreference(ctx); @@ -2643,36 +2637,63 @@ void i915_gem_request_free(struct kref *req_ref) } int i915_gem_request_alloc(struct intel_engine_cs *ring, - struct intel_context *ctx) + struct intel_context *ctx, + struct drm_i915_gem_request **req_out) { struct drm_i915_private *dev_priv = to_i915(ring->dev); struct drm_i915_gem_request *req; int ret; - if (ring->outstanding_lazy_request) - return 0; + if (!req_out) + return -EINVAL; + + *req_out = NULL; req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL); if (req == NULL) return -ENOMEM; - kref_init(&req->ref); - req->i915 = dev_priv; - ret = i915_gem_get_seqno(ring->dev, &req->seqno); if (ret) goto err; + kref_init(&req->ref); + req->i915 = dev_priv; req->ring = ring; + req->ctx = ctx; + i915_gem_context_reference(req->ctx); if (i915.enable_execlists) - ret = intel_logical_ring_alloc_request_extras(req, ctx); + ret = intel_logical_ring_alloc_request_extras(req); else ret = intel_ring_alloc_request_extras(req); - if (ret) + if (ret) { + i915_gem_context_unreference(req->ctx); goto err; + } - ring->outstanding_lazy_request = req; + /* + * Reserve space in the ring buffer for all the commands required to + * eventually emit this request. This is to guarantee that the + * i915_add_request() call can't fail. Note that the reserve may need + * to be redone if the request is not actually submitted straight + * away, e.g. because a GPU scheduler has deferred it. + */ + if (i915.enable_execlists) + ret = intel_logical_ring_reserve_space(req); + else + ret = intel_ring_reserve_space(req); + if (ret) { + /* + * At this point, the request is fully allocated even if not + * fully prepared. Thus it can be cleaned up using the proper + * free code. + */ + i915_gem_request_cancel(req); + return ret; + } + + *req_out = req; return 0; err: @@ -2680,6 +2701,13 @@ err: return ret; } +void i915_gem_request_cancel(struct drm_i915_gem_request *req) +{ + intel_ring_reserved_space_cancel(req->ringbuf); + + i915_gem_request_unreference(req); +} + struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *ring) { @@ -2717,6 +2745,8 @@ static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv, static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, struct intel_engine_cs *ring) { + struct intel_ringbuffer *buffer; + while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj; @@ -2732,18 +2762,23 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, * are the ones that keep the context and ringbuffer backing objects * pinned in place. */ - while (!list_empty(&ring->execlist_queue)) { - struct drm_i915_gem_request *submit_req; - submit_req = list_first_entry(&ring->execlist_queue, - struct drm_i915_gem_request, - execlist_link); - list_del(&submit_req->execlist_link); + if (i915.enable_execlists) { + spin_lock_irq(&ring->execlist_lock); + while (!list_empty(&ring->execlist_queue)) { + struct drm_i915_gem_request *submit_req; + + submit_req = list_first_entry(&ring->execlist_queue, + struct drm_i915_gem_request, + execlist_link); + list_del(&submit_req->execlist_link); - if (submit_req->ctx != ring->default_context) - intel_lr_context_unpin(ring, submit_req->ctx); + if (submit_req->ctx != ring->default_context) + intel_lr_context_unpin(submit_req); - i915_gem_request_unreference(submit_req); + i915_gem_request_unreference(submit_req); + } + spin_unlock_irq(&ring->execlist_lock); } /* @@ -2763,28 +2798,16 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, i915_gem_request_retire(request); } - /* This may not have been flushed before the reset, so clean it now */ - i915_gem_request_assign(&ring->outstanding_lazy_request, NULL); -} - -void i915_gem_restore_fences(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int i; - - for (i = 0; i < dev_priv->num_fence_regs; i++) { - struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; - - /* - * Commit delayed tiling changes if we have an object still - * attached to the fence, otherwise just clear the fence. - */ - if (reg->obj) { - i915_gem_object_update_fence(reg->obj, reg, - reg->obj->tiling_mode); - } else { - i915_gem_write_fence(dev, i, NULL); - } + /* Having flushed all requests from all queues, we know that all + * ringbuffers must now be empty. However, since we do not reclaim + * all space when retiring the request (to prevent HEADs colliding + * with rapid ringbuffer wraparound) the amount of available space + * upon reset is less than when we start. Do one more pass over + * all the ringbuffers to reset last_retired_head. + */ + list_for_each_entry(buffer, &ring->buffers, link) { + buffer->last_retired_head = buffer->tail; + intel_ring_update_space(buffer); } } @@ -2926,6 +2949,10 @@ i915_gem_idle_work_handler(struct work_struct *work) if (!list_empty(&ring->request_list)) return; + /* we probably should sync with hangcheck here, using cancel_work_sync. + * Also locking seems to be fubar here, ring->request_list is protected + * by dev->struct_mutex. */ + intel_mark_idle(dev); if (mutex_trylock(&dev->struct_mutex)) { @@ -2947,7 +2974,7 @@ i915_gem_idle_work_handler(struct work_struct *work) static int i915_gem_object_flush_active(struct drm_i915_gem_object *obj) { - int ret, i; + int i; if (!obj->active) return 0; @@ -2962,10 +2989,6 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj) if (list_empty(&req->list)) goto retire; - ret = i915_gem_check_olr(req); - if (ret) - return ret; - if (i915_gem_request_completed(req, true)) { __i915_gem_request_retire__upto(req); retire: @@ -3054,7 +3077,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (ret == 0) ret = __i915_wait_request(req[i], reset_counter, true, args->timeout_ns > 0 ? &args->timeout_ns : NULL, - file->driver_priv); + to_rps_client(file)); i915_gem_request_unreference__unlocked(req[i]); } return ret; @@ -3068,25 +3091,22 @@ out: static int __i915_gem_object_sync(struct drm_i915_gem_object *obj, struct intel_engine_cs *to, - struct drm_i915_gem_request *req) + struct drm_i915_gem_request *from_req, + struct drm_i915_gem_request **to_req) { struct intel_engine_cs *from; int ret; - from = i915_gem_request_get_ring(req); + from = i915_gem_request_get_ring(from_req); if (to == from) return 0; - if (i915_gem_request_completed(req, true)) + if (i915_gem_request_completed(from_req, true)) return 0; - ret = i915_gem_check_olr(req); - if (ret) - return ret; - if (!i915_semaphore_is_enabled(obj->base.dev)) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - ret = __i915_wait_request(req, + ret = __i915_wait_request(from_req, atomic_read(&i915->gpu_error.reset_counter), i915->mm.interruptible, NULL, @@ -3094,16 +3114,24 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj, if (ret) return ret; - i915_gem_object_retire_request(obj, req); + i915_gem_object_retire_request(obj, from_req); } else { int idx = intel_ring_sync_index(from, to); - u32 seqno = i915_gem_request_get_seqno(req); + u32 seqno = i915_gem_request_get_seqno(from_req); + + WARN_ON(!to_req); if (seqno <= from->semaphore.sync_seqno[idx]) return 0; - trace_i915_gem_ring_sync_to(from, to, req); - ret = to->semaphore.sync_to(to, from, seqno); + if (*to_req == NULL) { + ret = i915_gem_request_alloc(to, to->default_context, to_req); + if (ret) + return ret; + } + + trace_i915_gem_ring_sync_to(*to_req, from, from_req); + ret = to->semaphore.sync_to(*to_req, from, seqno); if (ret) return ret; @@ -3123,11 +3151,14 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj, * * @obj: object which may be in use on another ring. * @to: ring we wish to use the object on. May be NULL. + * @to_req: request we wish to use the object for. See below. + * This will be allocated and returned if a request is + * required but not passed in. * * This code is meant to abstract object synchronization with the GPU. * Calling with NULL implies synchronizing the object with the CPU * rather than a particular GPU ring. Conceptually we serialise writes - * between engines inside the GPU. We only allow on engine to write + * between engines inside the GPU. We only allow one engine to write * into a buffer at any time, but multiple readers. To ensure each has * a coherent view of memory, we must: * @@ -3138,11 +3169,22 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj, * - If we are a write request (pending_write_domain is set), the new * request must wait for outstanding read requests to complete. * + * For CPU synchronisation (NULL to) no request is required. For syncing with + * rings to_req must be non-NULL. However, a request does not have to be + * pre-allocated. If *to_req is NULL and sync commands will be emitted then a + * request will be allocated automatically and returned through *to_req. Note + * that it is not guaranteed that commands will be emitted (because the system + * might already be idle). Hence there is no need to create a request that + * might never have any work submitted. Note further that if a request is + * returned in *to_req, it is the responsibility of the caller to submit + * that request (after potentially adding more work to it). + * * Returns 0 if successful, else propagates up the lower layer error. */ int i915_gem_object_sync(struct drm_i915_gem_object *obj, - struct intel_engine_cs *to) + struct intel_engine_cs *to, + struct drm_i915_gem_request **to_req) { const bool readonly = obj->base.pending_write_domain == 0; struct drm_i915_gem_request *req[I915_NUM_RINGS]; @@ -3164,7 +3206,7 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj, req[n++] = obj->last_read_req[i]; } for (i = 0; i < n; i++) { - ret = __i915_gem_object_sync(obj, to, req[i]); + ret = __i915_gem_object_sync(obj, to, req[i], to_req); if (ret) return ret; } @@ -3196,7 +3238,7 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) old_write_domain); } -int i915_vma_unbind(struct i915_vma *vma) +static int __i915_vma_unbind(struct i915_vma *vma, bool wait) { struct drm_i915_gem_object *obj = vma->obj; struct drm_i915_private *dev_priv = obj->base.dev->dev_private; @@ -3215,13 +3257,11 @@ int i915_vma_unbind(struct i915_vma *vma) BUG_ON(obj->pages == NULL); - ret = i915_gem_object_wait_rendering(obj, false); - if (ret) - return ret; - /* Continue on if we fail due to EIO, the GPU is hung so we - * should be safe and we need to cleanup or else we might - * cause memory corruption through use-after-free. - */ + if (wait) { + ret = i915_gem_object_wait_rendering(obj, false); + if (ret) + return ret; + } if (i915_is_ggtt(vma->vm) && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { @@ -3266,6 +3306,16 @@ int i915_vma_unbind(struct i915_vma *vma) return 0; } +int i915_vma_unbind(struct i915_vma *vma) +{ + return __i915_vma_unbind(vma, true); +} + +int __i915_vma_unbind_no_wait(struct i915_vma *vma) +{ + return __i915_vma_unbind(vma, false); +} + int i915_gpu_idle(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -3275,354 +3325,27 @@ int i915_gpu_idle(struct drm_device *dev) /* Flush everything onto the inactive list. */ for_each_ring(ring, dev_priv, i) { if (!i915.enable_execlists) { - ret = i915_switch_context(ring, ring->default_context); + struct drm_i915_gem_request *req; + + ret = i915_gem_request_alloc(ring, ring->default_context, &req); if (ret) return ret; - } - ret = intel_ring_idle(ring); - if (ret) - return ret; - } - - WARN_ON(i915_verify_lists(dev)); - return 0; -} - -static void i965_write_fence_reg(struct drm_device *dev, int reg, - struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int fence_reg; - int fence_pitch_shift; - - if (INTEL_INFO(dev)->gen >= 6) { - fence_reg = FENCE_REG_SANDYBRIDGE_0; - fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT; - } else { - fence_reg = FENCE_REG_965_0; - fence_pitch_shift = I965_FENCE_PITCH_SHIFT; - } - - fence_reg += reg * 8; + ret = i915_switch_context(req); + if (ret) { + i915_gem_request_cancel(req); + return ret; + } - /* To w/a incoherency with non-atomic 64-bit register updates, - * we split the 64-bit update into two 32-bit writes. In order - * for a partial fence not to be evaluated between writes, we - * precede the update with write to turn off the fence register, - * and only enable the fence as the last step. - * - * For extra levels of paranoia, we make sure each step lands - * before applying the next step. - */ - I915_WRITE(fence_reg, 0); - POSTING_READ(fence_reg); - - if (obj) { - u32 size = i915_gem_obj_ggtt_size(obj); - uint64_t val; - - /* Adjust fence size to match tiled area */ - if (obj->tiling_mode != I915_TILING_NONE) { - uint32_t row_size = obj->stride * - (obj->tiling_mode == I915_TILING_Y ? 32 : 8); - size = (size / row_size) * row_size; + i915_add_request_no_flush(req); } - val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) & - 0xfffff000) << 32; - val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000; - val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I965_FENCE_TILING_Y_SHIFT; - val |= I965_FENCE_REG_VALID; - - I915_WRITE(fence_reg + 4, val >> 32); - POSTING_READ(fence_reg + 4); - - I915_WRITE(fence_reg + 0, val); - POSTING_READ(fence_reg); - } else { - I915_WRITE(fence_reg + 4, 0); - POSTING_READ(fence_reg + 4); - } -} - -static void i915_write_fence_reg(struct drm_device *dev, int reg, - struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - u32 val; - - if (obj) { - u32 size = i915_gem_obj_ggtt_size(obj); - int pitch_val; - int tile_width; - - WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) || - (size & -size) != size || - (i915_gem_obj_ggtt_offset(obj) & (size - 1)), - "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", - i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size); - - if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) - tile_width = 128; - else - tile_width = 512; - - /* Note: pitch better be a power of two tile widths */ - pitch_val = obj->stride / tile_width; - pitch_val = ffs(pitch_val) - 1; - - val = i915_gem_obj_ggtt_offset(obj); - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I830_FENCE_TILING_Y_SHIFT; - val |= I915_FENCE_SIZE_BITS(size); - val |= pitch_val << I830_FENCE_PITCH_SHIFT; - val |= I830_FENCE_REG_VALID; - } else - val = 0; - - if (reg < 8) - reg = FENCE_REG_830_0 + reg * 4; - else - reg = FENCE_REG_945_8 + (reg - 8) * 4; - - I915_WRITE(reg, val); - POSTING_READ(reg); -} - -static void i830_write_fence_reg(struct drm_device *dev, int reg, - struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t val; - - if (obj) { - u32 size = i915_gem_obj_ggtt_size(obj); - uint32_t pitch_val; - - WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) || - (size & -size) != size || - (i915_gem_obj_ggtt_offset(obj) & (size - 1)), - "object 0x%08lx not 512K or pot-size 0x%08x aligned\n", - i915_gem_obj_ggtt_offset(obj), size); - - pitch_val = obj->stride / 128; - pitch_val = ffs(pitch_val) - 1; - - val = i915_gem_obj_ggtt_offset(obj); - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I830_FENCE_TILING_Y_SHIFT; - val |= I830_FENCE_SIZE_BITS(size); - val |= pitch_val << I830_FENCE_PITCH_SHIFT; - val |= I830_FENCE_REG_VALID; - } else - val = 0; - - I915_WRITE(FENCE_REG_830_0 + reg * 4, val); - POSTING_READ(FENCE_REG_830_0 + reg * 4); -} - -inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj) -{ - return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT; -} - -static void i915_gem_write_fence(struct drm_device *dev, int reg, - struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - /* Ensure that all CPU reads are completed before installing a fence - * and all writes before removing the fence. - */ - if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) - mb(); - - WARN(obj && (!obj->stride || !obj->tiling_mode), - "bogus fence setup with stride: 0x%x, tiling mode: %i\n", - obj->stride, obj->tiling_mode); - - if (IS_GEN2(dev)) - i830_write_fence_reg(dev, reg, obj); - else if (IS_GEN3(dev)) - i915_write_fence_reg(dev, reg, obj); - else if (INTEL_INFO(dev)->gen >= 4) - i965_write_fence_reg(dev, reg, obj); - - /* And similarly be paranoid that no direct access to this region - * is reordered to before the fence is installed. - */ - if (i915_gem_object_needs_mb(obj)) - mb(); -} - -static inline int fence_number(struct drm_i915_private *dev_priv, - struct drm_i915_fence_reg *fence) -{ - return fence - dev_priv->fence_regs; -} - -static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, - struct drm_i915_fence_reg *fence, - bool enable) -{ - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - int reg = fence_number(dev_priv, fence); - - i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); - - if (enable) { - obj->fence_reg = reg; - fence->obj = obj; - list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); - } else { - obj->fence_reg = I915_FENCE_REG_NONE; - fence->obj = NULL; - list_del_init(&fence->lru_list); - } - obj->fence_dirty = false; -} - -static int -i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) -{ - if (obj->last_fenced_req) { - int ret = i915_wait_request(obj->last_fenced_req); - if (ret) - return ret; - - i915_gem_request_assign(&obj->last_fenced_req, NULL); - } - - return 0; -} - -int -i915_gem_object_put_fence(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - struct drm_i915_fence_reg *fence; - int ret; - - ret = i915_gem_object_wait_fence(obj); - if (ret) - return ret; - - if (obj->fence_reg == I915_FENCE_REG_NONE) - return 0; - - fence = &dev_priv->fence_regs[obj->fence_reg]; - - if (WARN_ON(fence->pin_count)) - return -EBUSY; - - i915_gem_object_fence_lost(obj); - i915_gem_object_update_fence(obj, fence, false); - - return 0; -} - -static struct drm_i915_fence_reg * -i915_find_fence_reg(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_fence_reg *reg, *avail; - int i; - - /* First try to find a free reg */ - avail = NULL; - for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { - reg = &dev_priv->fence_regs[i]; - if (!reg->obj) - return reg; - - if (!reg->pin_count) - avail = reg; - } - - if (avail == NULL) - goto deadlock; - - /* None available, try to steal one or wait for a user to finish */ - list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { - if (reg->pin_count) - continue; - - return reg; - } - -deadlock: - /* Wait for completion of pending flips which consume fences */ - if (intel_has_pending_fb_unpin(dev)) - return ERR_PTR(-EAGAIN); - - return ERR_PTR(-EDEADLK); -} - -/** - * i915_gem_object_get_fence - set up fencing for an object - * @obj: object to map through a fence reg - * - * When mapping objects through the GTT, userspace wants to be able to write - * to them without having to worry about swizzling if the object is tiled. - * This function walks the fence regs looking for a free one for @obj, - * stealing one if it can't find any. - * - * It then sets up the reg based on the object's properties: address, pitch - * and tiling format. - * - * For an untiled surface, this removes any existing fence. - */ -int -i915_gem_object_get_fence(struct drm_i915_gem_object *obj) -{ - struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - bool enable = obj->tiling_mode != I915_TILING_NONE; - struct drm_i915_fence_reg *reg; - int ret; - - /* Have we updated the tiling parameters upon the object and so - * will need to serialise the write to the associated fence register? - */ - if (obj->fence_dirty) { - ret = i915_gem_object_wait_fence(obj); + ret = intel_ring_idle(ring); if (ret) return ret; } - /* Just update our place in the LRU if our fence is getting reused. */ - if (obj->fence_reg != I915_FENCE_REG_NONE) { - reg = &dev_priv->fence_regs[obj->fence_reg]; - if (!obj->fence_dirty) { - list_move_tail(®->lru_list, - &dev_priv->mm.fence_list); - return 0; - } - } else if (enable) { - if (WARN_ON(!obj->map_and_fenceable)) - return -EINVAL; - - reg = i915_find_fence_reg(dev); - if (IS_ERR(reg)) - return PTR_ERR(reg); - - if (reg->obj) { - struct drm_i915_gem_object *old = reg->obj; - - ret = i915_gem_object_wait_fence(old); - if (ret) - return ret; - - i915_gem_object_fence_lost(old); - } - } else - return 0; - - i915_gem_object_update_fence(obj, reg, enable); - + WARN_ON(i915_verify_lists(dev)); return 0; } @@ -3672,11 +3395,10 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 size, fence_size, fence_alignment, unfenced_alignment; - unsigned long start = - flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; - unsigned long end = - flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total; + u32 fence_alignment, unfenced_alignment; + u32 search_flag, alloc_flag; + u64 start, end; + u64 size, fence_size; struct i915_vma *vma; int ret; @@ -3716,6 +3438,13 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; } + start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; + end = vm->total; + if (flags & PIN_MAPPABLE) + end = min_t(u64, end, dev_priv->gtt.mappable_end); + if (flags & PIN_ZONE_4G) + end = min_t(u64, end, (1ULL << 32)); + if (alignment == 0) alignment = flags & PIN_MAPPABLE ? fence_alignment : unfenced_alignment; @@ -3731,7 +3460,7 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, * attempt to find space. */ if (size > end) { - DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%u > %s aperture=%lu\n", + DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n", ggtt_view ? ggtt_view->type : 0, size, flags & PIN_MAPPABLE ? "mappable" : "total", @@ -3751,13 +3480,21 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, if (IS_ERR(vma)) goto err_unpin; + if (flags & PIN_HIGH) { + search_flag = DRM_MM_SEARCH_BELOW; + alloc_flag = DRM_MM_CREATE_TOP; + } else { + search_flag = DRM_MM_SEARCH_DEFAULT; + alloc_flag = DRM_MM_CREATE_DEFAULT; + } + search_free: ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, size, alignment, obj->cache_level, start, end, - DRM_MM_SEARCH_DEFAULT, - DRM_MM_CREATE_DEFAULT); + search_flag, + alloc_flag); if (ret) { ret = i915_gem_evict_something(dev, vm, size, alignment, obj->cache_level, @@ -3853,7 +3590,7 @@ i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; - intel_fb_obj_flush(obj, false); + intel_fb_obj_flush(obj, false, ORIGIN_GTT); trace_i915_gem_object_change_domain(obj, obj->base.read_domains, @@ -3875,7 +3612,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; - intel_fb_obj_flush(obj, false); + intel_fb_obj_flush(obj, false, ORIGIN_CPU); trace_i915_gem_object_change_domain(obj, obj->base.read_domains, @@ -3937,9 +3674,6 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) obj->dirty = 1; } - if (write) - intel_fb_obj_invalidate(obj, NULL, ORIGIN_GTT); - trace_i915_gem_object_change_domain(obj, old_read_domains, old_write_domain); @@ -3953,59 +3687,117 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) return 0; } +/** + * Changes the cache-level of an object across all VMA. + * + * After this function returns, the object will be in the new cache-level + * across all GTT and the contents of the backing storage will be coherent, + * with respect to the new cache-level. In order to keep the backing storage + * coherent for all users, we only allow a single cache level to be set + * globally on the object and prevent it from being changed whilst the + * hardware is reading from the object. That is if the object is currently + * on the scanout it will be set to uncached (or equivalent display + * cache coherency) and all non-MOCS GPU access will also be uncached so + * that all direct access to the scanout remains coherent. + */ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct drm_device *dev = obj->base.dev; struct i915_vma *vma, *next; - int ret; + bool bound = false; + int ret = 0; if (obj->cache_level == cache_level) - return 0; - - if (i915_gem_obj_is_pinned(obj)) { - DRM_DEBUG("can not change the cache level of pinned objects\n"); - return -EBUSY; - } + goto out; + /* Inspect the list of currently bound VMA and unbind any that would + * be invalid given the new cache-level. This is principally to + * catch the issue of the CS prefetch crossing page boundaries and + * reading an invalid PTE on older architectures. + */ list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { + if (!drm_mm_node_allocated(&vma->node)) + continue; + + if (vma->pin_count) { + DRM_DEBUG("can not change the cache level of pinned objects\n"); + return -EBUSY; + } + if (!i915_gem_valid_gtt_space(vma, cache_level)) { ret = i915_vma_unbind(vma); if (ret) return ret; - } + } else + bound = true; } - if (i915_gem_obj_bound_any(obj)) { + /* We can reuse the existing drm_mm nodes but need to change the + * cache-level on the PTE. We could simply unbind them all and + * rebind with the correct cache-level on next use. However since + * we already have a valid slot, dma mapping, pages etc, we may as + * rewrite the PTE in the belief that doing so tramples upon less + * state and so involves less work. + */ + if (bound) { + /* Before we change the PTE, the GPU must not be accessing it. + * If we wait upon the object, we know that all the bound + * VMA are no longer active. + */ ret = i915_gem_object_wait_rendering(obj, false); if (ret) return ret; - i915_gem_object_finish_gtt(obj); - - /* Before SandyBridge, you could not use tiling or fence - * registers with snooped memory, so relinquish any fences - * currently pointing to our region in the aperture. - */ - if (INTEL_INFO(dev)->gen < 6) { + if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) { + /* Access to snoopable pages through the GTT is + * incoherent and on some machines causes a hard + * lockup. Relinquish the CPU mmaping to force + * userspace to refault in the pages and we can + * then double check if the GTT mapping is still + * valid for that pointer access. + */ + i915_gem_release_mmap(obj); + + /* As we no longer need a fence for GTT access, + * we can relinquish it now (and so prevent having + * to steal a fence from someone else on the next + * fence request). Note GPU activity would have + * dropped the fence as all snoopable access is + * supposed to be linear. + */ ret = i915_gem_object_put_fence(obj); if (ret) return ret; + } else { + /* We either have incoherent backing store and + * so no GTT access or the architecture is fully + * coherent. In such cases, existing GTT mmaps + * ignore the cache bit in the PTE and we can + * rewrite it without confusing the GPU or having + * to force userspace to fault back in its mmaps. + */ } - list_for_each_entry(vma, &obj->vma_list, vma_link) - if (drm_mm_node_allocated(&vma->node)) { - ret = i915_vma_bind(vma, cache_level, - PIN_UPDATE); - if (ret) - return ret; - } + list_for_each_entry(vma, &obj->vma_list, vma_link) { + if (!drm_mm_node_allocated(&vma->node)) + continue; + + ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); + if (ret) + return ret; + } } list_for_each_entry(vma, &obj->vma_list, vma_link) vma->node.color = cache_level; obj->cache_level = cache_level; +out: + /* Flush the dirty CPU caches to the backing storage so that the + * object is now coherent at its new cache level (with respect + * to the access domain). + */ if (obj->cache_dirty && obj->base.write_domain != I915_GEM_DOMAIN_CPU && cpu_write_needs_clflush(obj)) { @@ -4048,6 +3840,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; enum i915_cache_level level; @@ -4058,6 +3851,15 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, level = I915_CACHE_NONE; break; case I915_CACHING_CACHED: + /* + * Due to a HW issue on BXT A stepping, GPU stores via a + * snooped mapping may leave stale data in a corresponding CPU + * cacheline, whereas normally such cachelines would get + * invalidated. + */ + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + return -ENODEV; + level = I915_CACHE_LLC; break; case I915_CACHING_DISPLAY: @@ -4067,9 +3869,11 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, return -EINVAL; } + intel_runtime_pm_get(dev_priv); + ret = i915_mutex_lock_interruptible(dev); if (ret) - return ret; + goto rpm_put; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { @@ -4082,6 +3886,9 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); +rpm_put: + intel_runtime_pm_put(dev_priv); + return ret; } @@ -4093,16 +3900,11 @@ unlock: int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, - struct intel_engine_cs *pipelined, const struct i915_ggtt_view *view) { u32 old_read_domains, old_write_domain; int ret; - ret = i915_gem_object_sync(obj, pipelined); - if (ret) - return ret; - /* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. */ @@ -4210,9 +4012,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) obj->base.write_domain = I915_GEM_DOMAIN_CPU; } - if (write) - intel_fb_obj_invalidate(obj, NULL, ORIGIN_CPU); - trace_i915_gem_object_change_domain(obj, old_read_domains, old_write_domain); @@ -4253,6 +4052,13 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) if (time_after_eq(request->emitted_jiffies, recent_enough)) break; + /* + * Note that the request might not have been submitted yet. + * In which case emitted_jiffies will be zero. + */ + if (!request->emitted_jiffies) + continue; + target = request; } reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); @@ -4326,15 +4132,13 @@ i915_gem_object_do_pin(struct drm_i915_gem_object *obj, return -EBUSY; if (i915_vma_misplaced(vma, alignment, flags)) { - unsigned long offset; - offset = ggtt_view ? i915_gem_obj_ggtt_offset_view(obj, ggtt_view) : - i915_gem_obj_offset(obj, vm); WARN(vma->pin_count, "bo is already pinned in %s with incorrect alignment:" - " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d," + " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d," " obj->map_and_fenceable=%d\n", ggtt_view ? "ggtt" : "ppgtt", - offset, + upper_32_bits(vma->node.start), + lower_32_bits(vma->node.start), alignment, !!(flags & PIN_MAPPABLE), obj->map_and_fenceable); @@ -4423,32 +4227,6 @@ i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, --vma->pin_count; } -bool -i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) -{ - if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj); - - WARN_ON(!ggtt_vma || - dev_priv->fence_regs[obj->fence_reg].pin_count > - ggtt_vma->pin_count); - dev_priv->fence_regs[obj->fence_reg].pin_count++; - return true; - } else - return false; -} - -void -i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) -{ - if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); - dev_priv->fence_regs[obj->fence_reg].pin_count--; - } -} - int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file) @@ -4723,10 +4501,8 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, { struct i915_vma *vma; list_for_each_entry(vma, &obj->vma_list, vma_link) { - if (i915_is_ggtt(vma->vm) && - vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) - continue; - if (vma->vm == vm) + if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL && + vma->vm == vm) return vma; } return NULL; @@ -4810,18 +4586,18 @@ err: return ret; } -int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice) +int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice) { + struct intel_engine_cs *ring = req->ring; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; int i, ret; if (!HAS_L3_DPF(dev) || !remap_info) return 0; - ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3); + ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3); if (ret) return ret; @@ -4830,10 +4606,10 @@ int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice) * here because no other code should access these registers other than * at initialization time. */ - for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { + for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, reg_base + i); - intel_ring_emit(ring, remap_info[i/4]); + intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i)); + intel_ring_emit(ring, remap_info[i]); } intel_ring_advance(ring); @@ -4866,22 +4642,6 @@ void i915_gem_init_swizzling(struct drm_device *dev) BUG(); } -static bool -intel_enable_blt(struct drm_device *dev) -{ - if (!HAS_BLT(dev)) - return false; - - /* The blitter was dysfunctional on early prototypes */ - if (IS_GEN6(dev) && dev->pdev->revision < 8) { - DRM_INFO("BLT not supported on this pre-production hardware;" - " graphics performance will be degraded.\n"); - return false; - } - - return true; -} - static void init_unused_ring(struct drm_device *dev, u32 base) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4924,7 +4684,7 @@ int i915_gem_init_rings(struct drm_device *dev) goto cleanup_render_ring; } - if (intel_enable_blt(dev)) { + if (HAS_BLT(dev)) { ret = intel_init_blt_ring_buffer(dev); if (ret) goto cleanup_bsd_ring; @@ -4942,14 +4702,8 @@ int i915_gem_init_rings(struct drm_device *dev) goto cleanup_vebox_ring; } - ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); - if (ret) - goto cleanup_bsd2_ring; - return 0; -cleanup_bsd2_ring: - intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]); cleanup_vebox_ring: intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); cleanup_blt_ring: @@ -4967,7 +4721,7 @@ i915_gem_init_hw(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring; - int ret, i; + int ret, i, j; if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) return -EIO; @@ -5004,27 +4758,73 @@ i915_gem_init_hw(struct drm_device *dev) */ init_unused_rings(dev); + BUG_ON(!dev_priv->ring[RCS].default_context); + + ret = i915_ppgtt_init_hw(dev); + if (ret) { + DRM_ERROR("PPGTT enable HW failed %d\n", ret); + goto out; + } + + /* Need to do basic initialisation of all rings first: */ for_each_ring(ring, dev_priv, i) { ret = ring->init_hw(ring); if (ret) goto out; } - for (i = 0; i < NUM_L3_SLICES(dev); i++) - i915_gem_l3_remap(&dev_priv->ring[RCS], i); - - ret = i915_ppgtt_init_hw(dev); - if (ret && ret != -EIO) { - DRM_ERROR("PPGTT enable failed %d\n", ret); - i915_gem_cleanup_ringbuffer(dev); + /* We can't enable contexts until all firmware is loaded */ + if (HAS_GUC_UCODE(dev)) { + ret = intel_guc_ucode_load(dev); + if (ret) { + DRM_ERROR("Failed to initialize GuC, error %d\n", ret); + ret = -EIO; + goto out; + } } - ret = i915_gem_context_enable(dev_priv); - if (ret && ret != -EIO) { - DRM_ERROR("Context enable failed %d\n", ret); - i915_gem_cleanup_ringbuffer(dev); - + /* + * Increment the next seqno by 0x100 so we have a visible break + * on re-initialisation + */ + ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100); + if (ret) goto out; + + /* Now it is safe to go back round and do everything else: */ + for_each_ring(ring, dev_priv, i) { + struct drm_i915_gem_request *req; + + WARN_ON(!ring->default_context); + + ret = i915_gem_request_alloc(ring, ring->default_context, &req); + if (ret) { + i915_gem_cleanup_ringbuffer(dev); + goto out; + } + + if (ring->id == RCS) { + for (j = 0; j < NUM_L3_SLICES(dev); j++) + i915_gem_l3_remap(req, j); + } + + ret = i915_ppgtt_init_ring(req); + if (ret && ret != -EIO) { + DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret); + i915_gem_request_cancel(req); + i915_gem_cleanup_ringbuffer(dev); + goto out; + } + + ret = i915_gem_context_enable(req); + if (ret && ret != -EIO) { + DRM_ERROR("Context enable ring #%d failed %d\n", i, ret); + i915_gem_request_cancel(req); + i915_gem_cleanup_ringbuffer(dev); + goto out; + } + + i915_add_request_no_flush(req); } out: @@ -5091,7 +4891,7 @@ int i915_gem_init(struct drm_device *dev) * for all other failure, such as an allocation failure, bail. */ DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); - atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); ret = 0; } @@ -5111,6 +4911,14 @@ i915_gem_cleanup_ringbuffer(struct drm_device *dev) for_each_ring(ring, dev_priv, i) dev_priv->gt.cleanup_ring(ring); + + if (i915.enable_execlists) + /* + * Neither the BIOS, ourselves or any other kernel + * expects the system to be in execlists mode on startup, + * so we need to reset the GPU back to legacy mode. + */ + intel_gpu_reset(dev); } static void @@ -5120,18 +4928,6 @@ init_ring_lists(struct intel_engine_cs *ring) INIT_LIST_HEAD(&ring->request_list); } -void i915_init_vm(struct drm_i915_private *dev_priv, - struct i915_address_space *vm) -{ - if (!i915_is_ggtt(vm)) - drm_mm_init(&vm->mm, vm->start, vm->total); - vm->dev = dev_priv->dev; - INIT_LIST_HEAD(&vm->active_list); - INIT_LIST_HEAD(&vm->inactive_list); - INIT_LIST_HEAD(&vm->global_link); - list_add_tail(&vm->global_link, &dev_priv->vm_list); -} - void i915_gem_load(struct drm_device *dev) { @@ -5155,8 +4951,6 @@ i915_gem_load(struct drm_device *dev) NULL); INIT_LIST_HEAD(&dev_priv->vm_list); - i915_init_vm(dev_priv, &dev_priv->gtt.base); - INIT_LIST_HEAD(&dev_priv->context_list); INIT_LIST_HEAD(&dev_priv->mm.unbound_list); INIT_LIST_HEAD(&dev_priv->mm.bound_list); @@ -5184,6 +4978,14 @@ i915_gem_load(struct drm_device *dev) dev_priv->num_fence_regs = I915_READ(vgtif_reg(avail_rs.fence_num)); + /* + * Set initial sequence number for requests. + * Using this number allows the wraparound to happen early, + * catching any obvious problems. + */ + dev_priv->next_seqno = ((u32)~0 - 0x1100); + dev_priv->last_seqno = ((u32)~0 - 0x1101); + /* Initialize fence registers to zero */ INIT_LIST_HEAD(&dev_priv->mm.fence_list); i915_gem_restore_fences(dev); @@ -5253,9 +5055,9 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file) /** * i915_gem_track_fb - update frontbuffer tracking - * old: current GEM buffer for the frontbuffer slots - * new: new GEM buffer for the frontbuffer slots - * frontbuffer_bits: bitmask of frontbuffer slots + * @old: current GEM buffer for the frontbuffer slots + * @new: new GEM buffer for the frontbuffer slots + * @frontbuffer_bits: bitmask of frontbuffer slots * * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them * from @old and setting them in @new. Both @old and @new can be NULL. @@ -5278,9 +5080,8 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, } /* All the new VM stuff */ -unsigned long -i915_gem_obj_offset(struct drm_i915_gem_object *o, - struct i915_address_space *vm) +u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, + struct i915_address_space *vm) { struct drm_i915_private *dev_priv = o->base.dev->dev_private; struct i915_vma *vma; @@ -5300,9 +5101,8 @@ i915_gem_obj_offset(struct drm_i915_gem_object *o, return -1; } -unsigned long -i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, - const struct i915_ggtt_view *view) +u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, + const struct i915_ggtt_view *view) { struct i915_address_space *ggtt = i915_obj_to_ggtt(o); struct i915_vma *vma; @@ -5388,3 +5188,42 @@ bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) return false; } +/* Allocate a new GEM object and fill it with the supplied data */ +struct drm_i915_gem_object * +i915_gem_object_create_from_data(struct drm_device *dev, + const void *data, size_t size) +{ + struct drm_i915_gem_object *obj; + struct sg_table *sg; + size_t bytes; + int ret; + + obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE)); + if (IS_ERR_OR_NULL(obj)) + return obj; + + ret = i915_gem_object_set_to_cpu_domain(obj, true); + if (ret) + goto fail; + + ret = i915_gem_object_get_pages(obj); + if (ret) + goto fail; + + i915_gem_object_pin_pages(obj); + sg = obj->pages; + bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); + i915_gem_object_unpin_pages(obj); + + if (WARN_ON(bytes != size)) { + DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size); + ret = -EFAULT; + goto fail; + } + + return obj; + +fail: + drm_gem_object_unreference(&obj->base); + return ERR_PTR(ret); +} diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 48afa777e94a..43761c5bcaca 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -133,6 +133,21 @@ static int get_context_size(struct drm_device *dev) return ret; } +static void i915_gem_context_clean(struct intel_context *ctx) +{ + struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; + struct i915_vma *vma, *next; + + if (!ppgtt) + return; + + list_for_each_entry_safe(vma, next, &ppgtt->base.inactive_list, + mm_list) { + if (WARN_ON(__i915_vma_unbind_no_wait(vma))) + break; + } +} + void i915_gem_context_free(struct kref *ctx_ref) { struct intel_context *ctx = container_of(ctx_ref, typeof(*ctx), ref); @@ -142,6 +157,13 @@ void i915_gem_context_free(struct kref *ctx_ref) if (i915.enable_execlists) intel_lr_context_free(ctx); + /* + * This context is going away and we need to remove all VMAs still + * around. This is to handle imported shared objects for which + * destructor did not run when their handles were closed. + */ + i915_gem_context_clean(ctx); + i915_ppgtt_put(ctx->ppgtt); if (ctx->legacy_hw_ctx.rcs_state) @@ -287,6 +309,7 @@ err_unpin: if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state); err_destroy: + idr_remove(&file_priv->context_idr, ctx->user_handle); i915_gem_context_unreference(ctx); return ERR_PTR(ret); } @@ -331,6 +354,13 @@ int i915_gem_context_init(struct drm_device *dev) if (WARN_ON(dev_priv->ring[RCS].default_context)) return 0; + if (intel_vgpu_active(dev) && HAS_LOGICAL_RING_CONTEXTS(dev)) { + if (!i915.enable_execlists) { + DRM_INFO("Only EXECLIST mode is supported in vgpu.\n"); + return -EINVAL; + } + } + if (i915.enable_execlists) { /* NB: intentionally left blank. We will allocate our own * backing objects as we need them, thank you very much */ @@ -407,32 +437,23 @@ void i915_gem_context_fini(struct drm_device *dev) i915_gem_context_unreference(dctx); } -int i915_gem_context_enable(struct drm_i915_private *dev_priv) +int i915_gem_context_enable(struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring; - int ret, i; - - BUG_ON(!dev_priv->ring[RCS].default_context); + struct intel_engine_cs *ring = req->ring; + int ret; if (i915.enable_execlists) { - for_each_ring(ring, dev_priv, i) { - if (ring->init_context) { - ret = ring->init_context(ring, - ring->default_context); - if (ret) { - DRM_ERROR("ring init context: %d\n", - ret); - return ret; - } - } - } + if (ring->init_context == NULL) + return 0; + ret = ring->init_context(req); } else - for_each_ring(ring, dev_priv, i) { - ret = i915_switch_context(ring, ring->default_context); - if (ret) - return ret; - } + ret = i915_switch_context(req); + + if (ret) { + DRM_ERROR("ring init context: %d\n", ret); + return ret; + } return 0; } @@ -485,10 +506,9 @@ i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id) } static inline int -mi_set_context(struct intel_engine_cs *ring, - struct intel_context *new_context, - u32 hw_flags) +mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) { + struct intel_engine_cs *ring = req->ring; u32 flags = hw_flags | MI_MM_SPACE_GTT; const int num_rings = /* Use an extended w/a on ivb+ if signalling from other rings */ @@ -503,13 +523,15 @@ mi_set_context(struct intel_engine_cs *ring, * itlb_before_ctx_switch. */ if (IS_GEN6(ring->dev)) { - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, 0); + ret = ring->flush(req, I915_GEM_GPU_DOMAINS, 0); if (ret) return ret; } /* These flags are for resource streamer on HSW+ */ - if (!IS_HASWELL(ring->dev) && INTEL_INFO(ring->dev)->gen < 8) + if (IS_HASWELL(ring->dev) || INTEL_INFO(ring->dev)->gen >= 8) + flags |= (HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN); + else if (INTEL_INFO(ring->dev)->gen < 8) flags |= (MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN); @@ -517,7 +539,7 @@ mi_set_context(struct intel_engine_cs *ring, if (INTEL_INFO(ring->dev)->gen >= 7) len += 2 + (num_rings ? 4*num_rings + 2 : 0); - ret = intel_ring_begin(ring, len); + ret = intel_ring_begin(req, len); if (ret) return ret; @@ -532,7 +554,7 @@ mi_set_context(struct intel_engine_cs *ring, if (signaller == ring) continue; - intel_ring_emit(ring, RING_PSMI_CTL(signaller->mmio_base)); + intel_ring_emit_reg(ring, RING_PSMI_CTL(signaller->mmio_base)); intel_ring_emit(ring, _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); } } @@ -540,7 +562,7 @@ mi_set_context(struct intel_engine_cs *ring, intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_SET_CONTEXT); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(new_context->legacy_hw_ctx.rcs_state) | + intel_ring_emit(ring, i915_gem_obj_ggtt_offset(req->ctx->legacy_hw_ctx.rcs_state) | flags); /* * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP @@ -557,7 +579,7 @@ mi_set_context(struct intel_engine_cs *ring, if (signaller == ring) continue; - intel_ring_emit(ring, RING_PSMI_CTL(signaller->mmio_base)); + intel_ring_emit_reg(ring, RING_PSMI_CTL(signaller->mmio_base)); intel_ring_emit(ring, _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); } } @@ -621,9 +643,10 @@ needs_pd_load_post(struct intel_engine_cs *ring, struct intel_context *to, return false; } -static int do_switch(struct intel_engine_cs *ring, - struct intel_context *to) +static int do_switch(struct drm_i915_gem_request *req) { + struct intel_context *to = req->ctx; + struct intel_engine_cs *ring = req->ring; struct drm_i915_private *dev_priv = ring->dev->dev_private; struct intel_context *from = ring->last_context; u32 hw_flags = 0; @@ -659,7 +682,7 @@ static int do_switch(struct intel_engine_cs *ring, * Register Immediate commands in Ring Buffer before submitting * a context."*/ trace_switch_mm(ring, to); - ret = to->ppgtt->switch_mm(to->ppgtt, ring); + ret = to->ppgtt->switch_mm(to->ppgtt, req); if (ret) goto unpin_out; @@ -701,7 +724,7 @@ static int do_switch(struct intel_engine_cs *ring, WARN_ON(needs_pd_load_pre(ring, to) && needs_pd_load_post(ring, to, hw_flags)); - ret = mi_set_context(ring, to, hw_flags); + ret = mi_set_context(req, hw_flags); if (ret) goto unpin_out; @@ -710,7 +733,7 @@ static int do_switch(struct intel_engine_cs *ring, */ if (needs_pd_load_post(ring, to, hw_flags)) { trace_switch_mm(ring, to); - ret = to->ppgtt->switch_mm(to->ppgtt, ring); + ret = to->ppgtt->switch_mm(to->ppgtt, req); /* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail * here. Still, let the user know something dangerous has @@ -726,7 +749,7 @@ static int do_switch(struct intel_engine_cs *ring, if (!(to->remap_slice & (1<<i))) continue; - ret = i915_gem_l3_remap(ring, i); + ret = i915_gem_l3_remap(req, i); /* If it failed, try again next round */ if (ret) DRM_DEBUG_DRIVER("L3 remapping failed\n"); @@ -742,7 +765,7 @@ static int do_switch(struct intel_engine_cs *ring, */ if (from != NULL) { from->legacy_hw_ctx.rcs_state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION; - i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->legacy_hw_ctx.rcs_state), ring); + i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->legacy_hw_ctx.rcs_state), req); /* As long as MI_SET_CONTEXT is serializing, ie. it flushes the * whole damn pipeline, we don't need to explicitly mark the * object dirty. The only exception is that the context must be @@ -766,7 +789,7 @@ done: if (uninitialized) { if (ring->init_context) { - ret = ring->init_context(ring, to); + ret = ring->init_context(req); if (ret) DRM_ERROR("ring init context: %d\n", ret); } @@ -782,8 +805,7 @@ unpin_out: /** * i915_switch_context() - perform a GPU context switch. - * @ring: ring for which we'll execute the context switch - * @to: the context to switch to + * @req: request for which we'll execute the context switch * * The context life cycle is simple. The context refcount is incremented and * decremented by 1 and create and destroy. If the context is in use by the GPU, @@ -794,25 +816,25 @@ unpin_out: * switched by writing to the ELSP and requests keep a reference to their * context. */ -int i915_switch_context(struct intel_engine_cs *ring, - struct intel_context *to) +int i915_switch_context(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; struct drm_i915_private *dev_priv = ring->dev->dev_private; WARN_ON(i915.enable_execlists); WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); - if (to->legacy_hw_ctx.rcs_state == NULL) { /* We have the fake context */ - if (to != ring->last_context) { - i915_gem_context_reference(to); + if (req->ctx->legacy_hw_ctx.rcs_state == NULL) { /* We have the fake context */ + if (req->ctx != ring->last_context) { + i915_gem_context_reference(req->ctx); if (ring->last_context) i915_gem_context_unreference(ring->last_context); - ring->last_context = to; + ring->last_context = req->ctx; } return 0; } - return do_switch(ring, to); + return do_switch(req); } static bool contexts_enabled(struct drm_device *dev) @@ -898,6 +920,17 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, case I915_CONTEXT_PARAM_BAN_PERIOD: args->value = ctx->hang_stats.ban_period_seconds; break; + case I915_CONTEXT_PARAM_NO_ZEROMAP: + args->value = ctx->flags & CONTEXT_NO_ZEROMAP; + break; + case I915_CONTEXT_PARAM_GTT_SIZE: + if (ctx->ppgtt) + args->value = ctx->ppgtt->base.total; + else if (to_i915(dev)->mm.aliasing_ppgtt) + args->value = to_i915(dev)->mm.aliasing_ppgtt->base.total; + else + args->value = to_i915(dev)->gtt.base.total; + break; default: ret = -EINVAL; break; @@ -935,6 +968,14 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, else ctx->hang_stats.ban_period_seconds = args->value; break; + case I915_CONTEXT_PARAM_NO_ZEROMAP: + if (args->size) { + ret = -EINVAL; + } else { + ctx->flags &= ~CONTEXT_NO_ZEROMAP; + ctx->flags |= args->value ? CONTEXT_NO_ZEROMAP : 0; + } + break; default: ret = -EINVAL; break; diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index d09e35ed9c9a..d71a133ceff5 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -237,48 +237,3 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle) return 0; } - -/** - * i915_gem_evict_everything - Try to evict all objects - * @dev: Device to evict objects for - * - * This functions tries to evict all gem objects from all address spaces. Used - * by the shrinker as a last-ditch effort and for suspend, before releasing the - * backing storage of all unbound objects. - */ -int -i915_gem_evict_everything(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_address_space *vm, *v; - bool lists_empty = true; - int ret; - - list_for_each_entry(vm, &dev_priv->vm_list, global_link) { - lists_empty = (list_empty(&vm->inactive_list) && - list_empty(&vm->active_list)); - if (!lists_empty) - lists_empty = false; - } - - if (lists_empty) - return -ENOSPC; - - trace_i915_gem_evict_everything(dev); - - /* The gpu_idle will flush everything in the write domain to the - * active list. Then we must move everything off the active list - * with retire requests. - */ - ret = i915_gpu_idle(dev); - if (ret) - return ret; - - i915_gem_retire_requests(dev); - - /* Having flushed everything, unbind() should never raise an error */ - list_for_each_entry_safe(vm, v, &dev_priv->vm_list, global_link) - WARN_ON(i915_gem_evict_vm(vm, false)); - - return 0; -} diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index a7fa14516cda..a4c243cec4aa 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -590,10 +590,17 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, flags |= PIN_GLOBAL; if (!drm_mm_node_allocated(&vma->node)) { + /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset, + * limit address to the first 4GBs for unflagged objects. + */ + if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0) + flags |= PIN_ZONE_4G; if (entry->flags & __EXEC_OBJECT_NEEDS_MAP) flags |= PIN_GLOBAL | PIN_MAPPABLE; if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; + if ((flags & PIN_MAPPABLE) == 0) + flags |= PIN_HIGH; } ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags); @@ -671,12 +678,17 @@ eb_vma_misplaced(struct i915_vma *vma) if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable) return !only_mappable_for_reloc(entry->flags); + if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 && + (vma->node.start + vma->node.size - 1) >> 32) + return true; + return false; } static int i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, struct list_head *vmas, + struct intel_context *ctx, bool *need_relocs) { struct drm_i915_gem_object *obj; @@ -699,6 +711,9 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, obj = vma->obj; entry = vma->exec_entry; + if (ctx->flags & CONTEXT_NO_ZEROMAP) + entry->flags |= __EXEC_OBJECT_NEEDS_BIAS; + if (!has_fenced_gpu_access) entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE; need_fence = @@ -776,7 +791,8 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, struct drm_file *file, struct intel_engine_cs *ring, struct eb_vmas *eb, - struct drm_i915_gem_exec_object2 *exec) + struct drm_i915_gem_exec_object2 *exec, + struct intel_context *ctx) { struct drm_i915_gem_relocation_entry *reloc; struct i915_address_space *vm; @@ -862,7 +878,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, goto err; need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; - ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs); + ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs); if (ret) goto err; @@ -887,10 +903,10 @@ err: } static int -i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, +i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, struct list_head *vmas) { - const unsigned other_rings = ~intel_ring_flag(ring); + const unsigned other_rings = ~intel_ring_flag(req->ring); struct i915_vma *vma; uint32_t flush_domains = 0; bool flush_chipset = false; @@ -900,7 +916,7 @@ i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, struct drm_i915_gem_object *obj = vma->obj; if (obj->active & other_rings) { - ret = i915_gem_object_sync(obj, ring); + ret = i915_gem_object_sync(obj, req->ring, &req); if (ret) return ret; } @@ -912,7 +928,7 @@ i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, } if (flush_chipset) - i915_gem_chipset_flush(ring->dev); + i915_gem_chipset_flush(req->ring->dev); if (flush_domains & I915_GEM_DOMAIN_GTT) wmb(); @@ -920,7 +936,7 @@ i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, /* Unconditionally invalidate gpu caches and ensure that we do flush * any residual writes from the previous batch. */ - return intel_ring_invalidate_all_caches(ring); + return intel_ring_invalidate_all_caches(req); } static bool @@ -929,7 +945,21 @@ i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS) return false; - return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0; + /* Kernel clipping was a DRI1 misfeature */ + if (exec->num_cliprects || exec->cliprects_ptr) + return false; + + if (exec->DR4 == 0xffffffff) { + DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); + exec->DR4 = 0; + } + if (exec->DR1 || exec->DR4) + return false; + + if ((exec->batch_start_offset | exec->batch_len) & 0x7) + return false; + + return true; } static int @@ -953,6 +983,9 @@ validate_exec_list(struct drm_device *dev, if (exec[i].flags & invalid_flags) return -EINVAL; + if (exec[i].alignment && !is_power_of_2(exec[i].alignment)) + return -EINVAL; + /* First check for malicious input causing overflow in * the worst case where we need to allocate the entire * relocation tree as a single array. @@ -1001,7 +1034,7 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, } if (i915.enable_execlists && !ctx->engine[ring->id].state) { - int ret = intel_lr_context_deferred_create(ctx, ring); + int ret = intel_lr_context_deferred_alloc(ctx, ring); if (ret) { DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret); return ERR_PTR(ret); @@ -1013,9 +1046,9 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, void i915_gem_execbuffer_move_to_active(struct list_head *vmas, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { - struct drm_i915_gem_request *req = intel_ring_get_request(ring); + struct intel_engine_cs *ring = i915_gem_request_get_ring(req); struct i915_vma *vma; list_for_each_entry(vma, vmas, exec_list) { @@ -1024,17 +1057,17 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas, u32 old_read = obj->base.read_domains; u32 old_write = obj->base.write_domain; + obj->dirty = 1; /* be paranoid */ obj->base.write_domain = obj->base.pending_write_domain; if (obj->base.write_domain == 0) obj->base.pending_read_domains |= obj->base.read_domains; obj->base.read_domains = obj->base.pending_read_domains; - i915_vma_move_to_active(vma, ring); + i915_vma_move_to_active(vma, req); if (obj->base.write_domain) { - obj->dirty = 1; i915_gem_request_assign(&obj->last_write_req, req); - intel_fb_obj_invalidate(obj, ring, ORIGIN_CS); + intel_fb_obj_invalidate(obj, ORIGIN_CS); /* update for the implicit flush after a batch */ obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; @@ -1053,22 +1086,20 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas, } void -i915_gem_execbuffer_retire_commands(struct drm_device *dev, - struct drm_file *file, - struct intel_engine_cs *ring, - struct drm_i915_gem_object *obj) +i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params) { /* Unconditionally force add_request to emit a full flush. */ - ring->gpu_caches_dirty = true; + params->ring->gpu_caches_dirty = true; /* Add a breadcrumb for the completion of the batch buffer */ - (void)__i915_add_request(ring, file, obj); + __i915_add_request(params->request, params->batch_obj, true); } static int i915_reset_gen7_sol_offsets(struct drm_device *dev, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; struct drm_i915_private *dev_priv = dev->dev_private; int ret, i; @@ -1077,13 +1108,13 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev, return -EINVAL; } - ret = intel_ring_begin(ring, 4 * 3); + ret = intel_ring_begin(req, 4 * 3); if (ret) return ret; for (i = 0; i < 4; i++) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i)); + intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i)); intel_ring_emit(ring, 0); } @@ -1092,46 +1123,6 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev, return 0; } -static int -i915_emit_box(struct intel_engine_cs *ring, - struct drm_clip_rect *box, - int DR1, int DR4) -{ - int ret; - - if (box->y2 <= box->y1 || box->x2 <= box->x1 || - box->y2 <= 0 || box->x2 <= 0) { - DRM_ERROR("Bad box %d,%d..%d,%d\n", - box->x1, box->y1, box->x2, box->y2); - return -EINVAL; - } - - if (INTEL_INFO(ring->dev)->gen >= 4) { - ret = intel_ring_begin(ring, 4); - if (ret) - return ret; - - intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO_I965); - intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16); - intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16); - intel_ring_emit(ring, DR4); - } else { - ret = intel_ring_begin(ring, 6); - if (ret) - return ret; - - intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO); - intel_ring_emit(ring, DR1); - intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16); - intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16); - intel_ring_emit(ring, DR4); - intel_ring_emit(ring, 0); - } - intel_ring_advance(ring); - - return 0; -} - static struct drm_i915_gem_object* i915_gem_execbuffer_parse(struct intel_engine_cs *ring, struct drm_i915_gem_exec_object2 *shadow_exec_entry, @@ -1186,73 +1177,27 @@ err: } int -i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, - struct intel_engine_cs *ring, - struct intel_context *ctx, +i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params, struct drm_i915_gem_execbuffer2 *args, - struct list_head *vmas, - struct drm_i915_gem_object *batch_obj, - u64 exec_start, u32 dispatch_flags) + struct list_head *vmas) { - struct drm_clip_rect *cliprects = NULL; + struct drm_device *dev = params->dev; + struct intel_engine_cs *ring = params->ring; struct drm_i915_private *dev_priv = dev->dev_private; - u64 exec_len; + u64 exec_start, exec_len; int instp_mode; u32 instp_mask; - int i, ret = 0; - - if (args->num_cliprects != 0) { - if (ring != &dev_priv->ring[RCS]) { - DRM_DEBUG("clip rectangles are only valid with the render ring\n"); - return -EINVAL; - } - - if (INTEL_INFO(dev)->gen >= 5) { - DRM_DEBUG("clip rectangles are only valid on pre-gen5\n"); - return -EINVAL; - } - - if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) { - DRM_DEBUG("execbuf with %u cliprects\n", - args->num_cliprects); - return -EINVAL; - } - - cliprects = kcalloc(args->num_cliprects, - sizeof(*cliprects), - GFP_KERNEL); - if (cliprects == NULL) { - ret = -ENOMEM; - goto error; - } - - if (copy_from_user(cliprects, - to_user_ptr(args->cliprects_ptr), - sizeof(*cliprects)*args->num_cliprects)) { - ret = -EFAULT; - goto error; - } - } else { - if (args->DR4 == 0xffffffff) { - DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); - args->DR4 = 0; - } - - if (args->DR1 || args->DR4 || args->cliprects_ptr) { - DRM_DEBUG("0 cliprects but dirt in cliprects fields\n"); - return -EINVAL; - } - } + int ret; - ret = i915_gem_execbuffer_move_to_gpu(ring, vmas); + ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas); if (ret) - goto error; + return ret; - ret = i915_switch_context(ring, ctx); + ret = i915_switch_context(params->request); if (ret) - goto error; + return ret; - WARN(ctx->ppgtt && ctx->ppgtt->pd_dirty_rings & (1<<ring->id), + WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<<ring->id), "%s didn't clear reload\n", ring->name); instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK; @@ -1263,22 +1208,19 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, case I915_EXEC_CONSTANTS_REL_SURFACE: if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) { DRM_DEBUG("non-0 rel constants mode on non-RCS\n"); - ret = -EINVAL; - goto error; + return -EINVAL; } if (instp_mode != dev_priv->relative_constants_mode) { if (INTEL_INFO(dev)->gen < 4) { DRM_DEBUG("no rel constants on pre-gen4\n"); - ret = -EINVAL; - goto error; + return -EINVAL; } if (INTEL_INFO(dev)->gen > 5 && instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) { DRM_DEBUG("rel surface constants mode invalid on gen5+\n"); - ret = -EINVAL; - goto error; + return -EINVAL; } /* The HW changed the meaning on this bit on gen6 */ @@ -1288,19 +1230,18 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, break; default: DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode); - ret = -EINVAL; - goto error; + return -EINVAL; } if (ring == &dev_priv->ring[RCS] && - instp_mode != dev_priv->relative_constants_mode) { - ret = intel_ring_begin(ring, 4); + instp_mode != dev_priv->relative_constants_mode) { + ret = intel_ring_begin(params->request, 4); if (ret) - goto error; + return ret; intel_ring_emit(ring, MI_NOOP); intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, INSTPM); + intel_ring_emit_reg(ring, INSTPM); intel_ring_emit(ring, instp_mask << 16 | instp_mode); intel_ring_advance(ring); @@ -1308,41 +1249,27 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file, } if (args->flags & I915_EXEC_GEN7_SOL_RESET) { - ret = i915_reset_gen7_sol_offsets(dev, ring); + ret = i915_reset_gen7_sol_offsets(dev, params->request); if (ret) - goto error; + return ret; } - exec_len = args->batch_len; - if (cliprects) { - for (i = 0; i < args->num_cliprects; i++) { - ret = i915_emit_box(ring, &cliprects[i], - args->DR1, args->DR4); - if (ret) - goto error; + exec_len = args->batch_len; + exec_start = params->batch_obj_vm_offset + + params->args_batch_start_offset; - ret = ring->dispatch_execbuffer(ring, - exec_start, exec_len, - dispatch_flags); - if (ret) - goto error; - } - } else { - ret = ring->dispatch_execbuffer(ring, - exec_start, exec_len, - dispatch_flags); - if (ret) - return ret; - } + ret = ring->dispatch_execbuffer(params->request, + exec_start, exec_len, + params->dispatch_flags); + if (ret) + return ret; - trace_i915_gem_ring_dispatch(intel_ring_get_request(ring), dispatch_flags); + trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags); - i915_gem_execbuffer_move_to_active(vmas, ring); - i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); + i915_gem_execbuffer_move_to_active(vmas, params->request); + i915_gem_execbuffer_retire_commands(params); -error: - kfree(cliprects); - return ret; + return 0; } /** @@ -1408,8 +1335,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct intel_engine_cs *ring; struct intel_context *ctx; struct i915_address_space *vm; + struct i915_execbuffer_params params_master; /* XXX: will be removed later */ + struct i915_execbuffer_params *params = ¶ms_master; const u32 ctx_id = i915_execbuffer2_get_context_id(*args); - u64 exec_start = args->batch_start_offset; u32 dispatch_flags; int ret; bool need_relocs; @@ -1482,6 +1410,20 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, return -EINVAL; } + if (args->flags & I915_EXEC_RESOURCE_STREAMER) { + if (!HAS_RESOURCE_STREAMER(dev)) { + DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n"); + return -EINVAL; + } + if (ring->id != RCS) { + DRM_DEBUG("RS is not available on %s\n", + ring->name); + return -EINVAL; + } + + dispatch_flags |= I915_DISPATCH_RS; + } + intel_runtime_pm_get(dev_priv); ret = i915_mutex_lock_interruptible(dev); @@ -1502,6 +1444,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, else vm = &dev_priv->gtt.base; + memset(¶ms_master, 0x00, sizeof(params_master)); + eb = eb_create(args); if (eb == NULL) { i915_gem_context_unreference(ctx); @@ -1520,7 +1464,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, /* Move the objects en-masse into the GTT, evicting if necessary. */ need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; - ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs); + ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs); if (ret) goto err; @@ -1530,7 +1474,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, if (ret) { if (ret == -EFAULT) { ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring, - eb, exec); + eb, exec, ctx); BUG_ON(!mutex_is_locked(&dev->struct_mutex)); } if (ret) @@ -1544,6 +1488,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, goto err; } + params->args_batch_start_offset = args->batch_start_offset; if (i915_needs_cmd_parser(ring) && args->batch_len) { struct drm_i915_gem_object *parsed_batch_obj; @@ -1575,7 +1520,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, * command parser has accepted. */ dispatch_flags |= I915_DISPATCH_SECURE; - exec_start = 0; + params->args_batch_start_offset = 0; batch_obj = parsed_batch_obj; } } @@ -1600,14 +1545,35 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, if (ret) goto err; - exec_start += i915_gem_obj_ggtt_offset(batch_obj); + params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj); } else - exec_start += i915_gem_obj_offset(batch_obj, vm); + params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm); + + /* Allocate a request for this batch buffer nice and early. */ + ret = i915_gem_request_alloc(ring, ctx, ¶ms->request); + if (ret) + goto err_batch_unpin; + + ret = i915_gem_request_add_to_client(params->request, file); + if (ret) + goto err_batch_unpin; + + /* + * Save assorted stuff away to pass through to *_submission(). + * NB: This data should be 'persistent' and not local as it will + * kept around beyond the duration of the IOCTL once the GPU + * scheduler arrives. + */ + params->dev = dev; + params->file = file; + params->ring = ring; + params->dispatch_flags = dispatch_flags; + params->batch_obj = batch_obj; + params->ctx = ctx; - ret = dev_priv->gt.execbuf_submit(dev, file, ring, ctx, args, - &eb->vmas, batch_obj, exec_start, - dispatch_flags); + ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas); +err_batch_unpin: /* * FIXME: We crucially rely upon the active tracking for the (ppgtt) * batch vma for correctness. For less ugly and less fragility this @@ -1616,11 +1582,20 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, */ if (dispatch_flags & I915_DISPATCH_SECURE) i915_gem_object_ggtt_unpin(batch_obj); + err: /* the request owns the ref now */ i915_gem_context_unreference(ctx); eb_destroy(eb); + /* + * If the request was created but not successfully submitted then it + * must be freed again. If it was submitted then it is being tracked + * on the active request list and no clean up is required here. + */ + if (ret && params->request) + i915_gem_request_cancel(params->request); + mutex_unlock(&dev->struct_mutex); pre_mutex_err: diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c new file mode 100644 index 000000000000..598198543dcd --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_fence.c @@ -0,0 +1,800 @@ +/* + * Copyright © 2008-2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include <drm/drmP.h> +#include <drm/i915_drm.h> +#include "i915_drv.h" + +/** + * DOC: fence register handling + * + * Important to avoid confusions: "fences" in the i915 driver are not execution + * fences used to track command completion but hardware detiler objects which + * wrap a given range of the global GTT. Each platform has only a fairly limited + * set of these objects. + * + * Fences are used to detile GTT memory mappings. They're also connected to the + * hardware frontbuffer render tracking and hence interract with frontbuffer + * conmpression. Furthermore on older platforms fences are required for tiled + * objects used by the display engine. They can also be used by the render + * engine - they're required for blitter commands and are optional for render + * commands. But on gen4+ both display (with the exception of fbc) and rendering + * have their own tiling state bits and don't need fences. + * + * Also note that fences only support X and Y tiling and hence can't be used for + * the fancier new tiling formats like W, Ys and Yf. + * + * Finally note that because fences are such a restricted resource they're + * dynamically associated with objects. Furthermore fence state is committed to + * the hardware lazily to avoid unecessary stalls on gen2/3. Therefore code must + * explictly call i915_gem_object_get_fence() to synchronize fencing status + * for cpu access. Also note that some code wants an unfenced view, for those + * cases the fence can be removed forcefully with i915_gem_object_put_fence(). + * + * Internally these functions will synchronize with userspace access by removing + * CPU ptes into GTT mmaps (not the GTT ptes themselves) as needed. + */ + +static void i965_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + i915_reg_t fence_reg_lo, fence_reg_hi; + int fence_pitch_shift; + + if (INTEL_INFO(dev)->gen >= 6) { + fence_reg_lo = FENCE_REG_GEN6_LO(reg); + fence_reg_hi = FENCE_REG_GEN6_HI(reg); + fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT; + } else { + fence_reg_lo = FENCE_REG_965_LO(reg); + fence_reg_hi = FENCE_REG_965_HI(reg); + fence_pitch_shift = I965_FENCE_PITCH_SHIFT; + } + + /* To w/a incoherency with non-atomic 64-bit register updates, + * we split the 64-bit update into two 32-bit writes. In order + * for a partial fence not to be evaluated between writes, we + * precede the update with write to turn off the fence register, + * and only enable the fence as the last step. + * + * For extra levels of paranoia, we make sure each step lands + * before applying the next step. + */ + I915_WRITE(fence_reg_lo, 0); + POSTING_READ(fence_reg_lo); + + if (obj) { + u32 size = i915_gem_obj_ggtt_size(obj); + uint64_t val; + + /* Adjust fence size to match tiled area */ + if (obj->tiling_mode != I915_TILING_NONE) { + uint32_t row_size = obj->stride * + (obj->tiling_mode == I915_TILING_Y ? 32 : 8); + size = (size / row_size) * row_size; + } + + val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) & + 0xfffff000) << 32; + val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000; + val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I965_FENCE_TILING_Y_SHIFT; + val |= I965_FENCE_REG_VALID; + + I915_WRITE(fence_reg_hi, val >> 32); + POSTING_READ(fence_reg_hi); + + I915_WRITE(fence_reg_lo, val); + POSTING_READ(fence_reg_lo); + } else { + I915_WRITE(fence_reg_hi, 0); + POSTING_READ(fence_reg_hi); + } +} + +static void i915_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; + + if (obj) { + u32 size = i915_gem_obj_ggtt_size(obj); + int pitch_val; + int tile_width; + + WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) || + (size & -size) != size || + (i915_gem_obj_ggtt_offset(obj) & (size - 1)), + "object 0x%08llx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", + i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size); + + if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) + tile_width = 128; + else + tile_width = 512; + + /* Note: pitch better be a power of two tile widths */ + pitch_val = obj->stride / tile_width; + pitch_val = ffs(pitch_val) - 1; + + val = i915_gem_obj_ggtt_offset(obj); + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I830_FENCE_TILING_Y_SHIFT; + val |= I915_FENCE_SIZE_BITS(size); + val |= pitch_val << I830_FENCE_PITCH_SHIFT; + val |= I830_FENCE_REG_VALID; + } else + val = 0; + + I915_WRITE(FENCE_REG(reg), val); + POSTING_READ(FENCE_REG(reg)); +} + +static void i830_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t val; + + if (obj) { + u32 size = i915_gem_obj_ggtt_size(obj); + uint32_t pitch_val; + + WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) || + (size & -size) != size || + (i915_gem_obj_ggtt_offset(obj) & (size - 1)), + "object 0x%08llx not 512K or pot-size 0x%08x aligned\n", + i915_gem_obj_ggtt_offset(obj), size); + + pitch_val = obj->stride / 128; + pitch_val = ffs(pitch_val) - 1; + + val = i915_gem_obj_ggtt_offset(obj); + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I830_FENCE_TILING_Y_SHIFT; + val |= I830_FENCE_SIZE_BITS(size); + val |= pitch_val << I830_FENCE_PITCH_SHIFT; + val |= I830_FENCE_REG_VALID; + } else + val = 0; + + I915_WRITE(FENCE_REG(reg), val); + POSTING_READ(FENCE_REG(reg)); +} + +inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj) +{ + return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT; +} + +static void i915_gem_write_fence(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + /* Ensure that all CPU reads are completed before installing a fence + * and all writes before removing the fence. + */ + if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) + mb(); + + WARN(obj && (!obj->stride || !obj->tiling_mode), + "bogus fence setup with stride: 0x%x, tiling mode: %i\n", + obj->stride, obj->tiling_mode); + + if (IS_GEN2(dev)) + i830_write_fence_reg(dev, reg, obj); + else if (IS_GEN3(dev)) + i915_write_fence_reg(dev, reg, obj); + else if (INTEL_INFO(dev)->gen >= 4) + i965_write_fence_reg(dev, reg, obj); + + /* And similarly be paranoid that no direct access to this region + * is reordered to before the fence is installed. + */ + if (i915_gem_object_needs_mb(obj)) + mb(); +} + +static inline int fence_number(struct drm_i915_private *dev_priv, + struct drm_i915_fence_reg *fence) +{ + return fence - dev_priv->fence_regs; +} + +static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, + struct drm_i915_fence_reg *fence, + bool enable) +{ + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + int reg = fence_number(dev_priv, fence); + + i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); + + if (enable) { + obj->fence_reg = reg; + fence->obj = obj; + list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); + } else { + obj->fence_reg = I915_FENCE_REG_NONE; + fence->obj = NULL; + list_del_init(&fence->lru_list); + } + obj->fence_dirty = false; +} + +static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) +{ + if (obj->tiling_mode) + i915_gem_release_mmap(obj); + + /* As we do not have an associated fence register, we will force + * a tiling change if we ever need to acquire one. + */ + obj->fence_dirty = false; + obj->fence_reg = I915_FENCE_REG_NONE; +} + +static int +i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) +{ + if (obj->last_fenced_req) { + int ret = i915_wait_request(obj->last_fenced_req); + if (ret) + return ret; + + i915_gem_request_assign(&obj->last_fenced_req, NULL); + } + + return 0; +} + +/** + * i915_gem_object_put_fence - force-remove fence for an object + * @obj: object to map through a fence reg + * + * This function force-removes any fence from the given object, which is useful + * if the kernel wants to do untiled GTT access. + * + * Returns: + * + * 0 on success, negative error code on failure. + */ +int +i915_gem_object_put_fence(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_fence_reg *fence; + int ret; + + ret = i915_gem_object_wait_fence(obj); + if (ret) + return ret; + + if (obj->fence_reg == I915_FENCE_REG_NONE) + return 0; + + fence = &dev_priv->fence_regs[obj->fence_reg]; + + if (WARN_ON(fence->pin_count)) + return -EBUSY; + + i915_gem_object_fence_lost(obj); + i915_gem_object_update_fence(obj, fence, false); + + return 0; +} + +static struct drm_i915_fence_reg * +i915_find_fence_reg(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_fence_reg *reg, *avail; + int i; + + /* First try to find a free reg */ + avail = NULL; + for (i = 0; i < dev_priv->num_fence_regs; i++) { + reg = &dev_priv->fence_regs[i]; + if (!reg->obj) + return reg; + + if (!reg->pin_count) + avail = reg; + } + + if (avail == NULL) + goto deadlock; + + /* None available, try to steal one or wait for a user to finish */ + list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { + if (reg->pin_count) + continue; + + return reg; + } + +deadlock: + /* Wait for completion of pending flips which consume fences */ + if (intel_has_pending_fb_unpin(dev)) + return ERR_PTR(-EAGAIN); + + return ERR_PTR(-EDEADLK); +} + +/** + * i915_gem_object_get_fence - set up fencing for an object + * @obj: object to map through a fence reg + * + * When mapping objects through the GTT, userspace wants to be able to write + * to them without having to worry about swizzling if the object is tiled. + * This function walks the fence regs looking for a free one for @obj, + * stealing one if it can't find any. + * + * It then sets up the reg based on the object's properties: address, pitch + * and tiling format. + * + * For an untiled surface, this removes any existing fence. + * + * Returns: + * + * 0 on success, negative error code on failure. + */ +int +i915_gem_object_get_fence(struct drm_i915_gem_object *obj) +{ + struct drm_device *dev = obj->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + bool enable = obj->tiling_mode != I915_TILING_NONE; + struct drm_i915_fence_reg *reg; + int ret; + + /* Have we updated the tiling parameters upon the object and so + * will need to serialise the write to the associated fence register? + */ + if (obj->fence_dirty) { + ret = i915_gem_object_wait_fence(obj); + if (ret) + return ret; + } + + /* Just update our place in the LRU if our fence is getting reused. */ + if (obj->fence_reg != I915_FENCE_REG_NONE) { + reg = &dev_priv->fence_regs[obj->fence_reg]; + if (!obj->fence_dirty) { + list_move_tail(®->lru_list, + &dev_priv->mm.fence_list); + return 0; + } + } else if (enable) { + if (WARN_ON(!obj->map_and_fenceable)) + return -EINVAL; + + reg = i915_find_fence_reg(dev); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + if (reg->obj) { + struct drm_i915_gem_object *old = reg->obj; + + ret = i915_gem_object_wait_fence(old); + if (ret) + return ret; + + i915_gem_object_fence_lost(old); + } + } else + return 0; + + i915_gem_object_update_fence(obj, reg, enable); + + return 0; +} + +/** + * i915_gem_object_pin_fence - pin fencing state + * @obj: object to pin fencing for + * + * This pins the fencing state (whether tiled or untiled) to make sure the + * object is ready to be used as a scanout target. Fencing status must be + * synchronize first by calling i915_gem_object_get_fence(): + * + * The resulting fence pin reference must be released again with + * i915_gem_object_unpin_fence(). + * + * Returns: + * + * True if the object has a fence, false otherwise. + */ +bool +i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) +{ + if (obj->fence_reg != I915_FENCE_REG_NONE) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj); + + WARN_ON(!ggtt_vma || + dev_priv->fence_regs[obj->fence_reg].pin_count > + ggtt_vma->pin_count); + dev_priv->fence_regs[obj->fence_reg].pin_count++; + return true; + } else + return false; +} + +/** + * i915_gem_object_unpin_fence - unpin fencing state + * @obj: object to unpin fencing for + * + * This releases the fence pin reference acquired through + * i915_gem_object_pin_fence. It will handle both objects with and without an + * attached fence correctly, callers do not need to distinguish this. + */ +void +i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) +{ + if (obj->fence_reg != I915_FENCE_REG_NONE) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); + dev_priv->fence_regs[obj->fence_reg].pin_count--; + } +} + +/** + * i915_gem_restore_fences - restore fence state + * @dev: DRM device + * + * Restore the hw fence state to match the software tracking again, to be called + * after a gpu reset and on resume. + */ +void i915_gem_restore_fences(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int i; + + for (i = 0; i < dev_priv->num_fence_regs; i++) { + struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; + + /* + * Commit delayed tiling changes if we have an object still + * attached to the fence, otherwise just clear the fence. + */ + if (reg->obj) { + i915_gem_object_update_fence(reg->obj, reg, + reg->obj->tiling_mode); + } else { + i915_gem_write_fence(dev, i, NULL); + } + } +} + +/** + * DOC: tiling swizzling details + * + * The idea behind tiling is to increase cache hit rates by rearranging + * pixel data so that a group of pixel accesses are in the same cacheline. + * Performance improvement from doing this on the back/depth buffer are on + * the order of 30%. + * + * Intel architectures make this somewhat more complicated, though, by + * adjustments made to addressing of data when the memory is in interleaved + * mode (matched pairs of DIMMS) to improve memory bandwidth. + * For interleaved memory, the CPU sends every sequential 64 bytes + * to an alternate memory channel so it can get the bandwidth from both. + * + * The GPU also rearranges its accesses for increased bandwidth to interleaved + * memory, and it matches what the CPU does for non-tiled. However, when tiled + * it does it a little differently, since one walks addresses not just in the + * X direction but also Y. So, along with alternating channels when bit + * 6 of the address flips, it also alternates when other bits flip -- Bits 9 + * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines) + * are common to both the 915 and 965-class hardware. + * + * The CPU also sometimes XORs in higher bits as well, to improve + * bandwidth doing strided access like we do so frequently in graphics. This + * is called "Channel XOR Randomization" in the MCH documentation. The result + * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address + * decode. + * + * All of this bit 6 XORing has an effect on our memory management, + * as we need to make sure that the 3d driver can correctly address object + * contents. + * + * If we don't have interleaved memory, all tiling is safe and no swizzling is + * required. + * + * When bit 17 is XORed in, we simply refuse to tile at all. Bit + * 17 is not just a page offset, so as we page an objet out and back in, + * individual pages in it will have different bit 17 addresses, resulting in + * each 64 bytes being swapped with its neighbor! + * + * Otherwise, if interleaved, we have to tell the 3d driver what the address + * swizzling it needs to do is, since it's writing with the CPU to the pages + * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the + * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling + * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order + * to match what the GPU expects. + */ + +/** + * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern + * @dev: DRM device + * + * Detects bit 6 swizzling of address lookup between IGD access and CPU + * access through main memory. + */ +void +i915_gem_detect_bit_6_swizzle(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; + uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; + + if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) { + /* + * On BDW+, swizzling is not used. We leave the CPU memory + * controller in charge of optimizing memory accesses without + * the extra address manipulation GPU side. + * + * VLV and CHV don't have GPU swizzling. + */ + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } else if (INTEL_INFO(dev)->gen >= 6) { + if (dev_priv->preserve_bios_swizzle) { + if (I915_READ(DISP_ARB_CTL) & + DISP_TILE_SURFACE_SWIZZLING) { + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } else { + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } + } else { + uint32_t dimm_c0, dimm_c1; + dimm_c0 = I915_READ(MAD_DIMM_C0); + dimm_c1 = I915_READ(MAD_DIMM_C1); + dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; + dimm_c1 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; + /* Enable swizzling when the channels are populated + * with identically sized dimms. We don't need to check + * the 3rd channel because no cpu with gpu attached + * ships in that configuration. Also, swizzling only + * makes sense for 2 channels anyway. */ + if (dimm_c0 == dimm_c1) { + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } else { + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } + } + } else if (IS_GEN5(dev)) { + /* On Ironlake whatever DRAM config, GPU always do + * same swizzling setup. + */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } else if (IS_GEN2(dev)) { + /* As far as we know, the 865 doesn't have these bit 6 + * swizzling issues. + */ + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } else if (IS_MOBILE(dev) || (IS_GEN3(dev) && !IS_G33(dev))) { + uint32_t dcc; + + /* On 9xx chipsets, channel interleave by the CPU is + * determined by DCC. For single-channel, neither the CPU + * nor the GPU do swizzling. For dual channel interleaved, + * the GPU's interleave is bit 9 and 10 for X tiled, and bit + * 9 for Y tiled. The CPU's interleave is independent, and + * can be based on either bit 11 (haven't seen this yet) or + * bit 17 (common). + */ + dcc = I915_READ(DCC); + switch (dcc & DCC_ADDRESSING_MODE_MASK) { + case DCC_ADDRESSING_MODE_SINGLE_CHANNEL: + case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC: + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + break; + case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED: + if (dcc & DCC_CHANNEL_XOR_DISABLE) { + /* This is the base swizzling by the GPU for + * tiled buffers. + */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) { + /* Bit 11 swizzling by the CPU in addition. */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10_11; + swizzle_y = I915_BIT_6_SWIZZLE_9_11; + } else { + /* Bit 17 swizzling by the CPU in addition. */ + swizzle_x = I915_BIT_6_SWIZZLE_9_10_17; + swizzle_y = I915_BIT_6_SWIZZLE_9_17; + } + break; + } + + /* check for L-shaped memory aka modified enhanced addressing */ + if (IS_GEN4(dev) && + !(I915_READ(DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) { + swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; + swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; + } + + if (dcc == 0xffffffff) { + DRM_ERROR("Couldn't read from MCHBAR. " + "Disabling tiling.\n"); + swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; + swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; + } + } else { + /* The 965, G33, and newer, have a very flexible memory + * configuration. It will enable dual-channel mode + * (interleaving) on as much memory as it can, and the GPU + * will additionally sometimes enable different bit 6 + * swizzling for tiled objects from the CPU. + * + * Here's what I found on the G965: + * slot fill memory size swizzling + * 0A 0B 1A 1B 1-ch 2-ch + * 512 0 0 0 512 0 O + * 512 0 512 0 16 1008 X + * 512 0 0 512 16 1008 X + * 0 512 0 512 16 1008 X + * 1024 1024 1024 0 2048 1024 O + * + * We could probably detect this based on either the DRB + * matching, which was the case for the swizzling required in + * the table above, or from the 1-ch value being less than + * the minimum size of a rank. + * + * Reports indicate that the swizzling actually + * varies depending upon page placement inside the + * channels, i.e. we see swizzled pages where the + * banks of memory are paired and unswizzled on the + * uneven portion, so leave that as unknown. + */ + if (I915_READ16(C0DRB3) == I915_READ16(C1DRB3)) { + swizzle_x = I915_BIT_6_SWIZZLE_9_10; + swizzle_y = I915_BIT_6_SWIZZLE_9; + } + } + + if (swizzle_x == I915_BIT_6_SWIZZLE_UNKNOWN || + swizzle_y == I915_BIT_6_SWIZZLE_UNKNOWN) { + /* Userspace likes to explode if it sees unknown swizzling, + * so lie. We will finish the lie when reporting through + * the get-tiling-ioctl by reporting the physical swizzle + * mode as unknown instead. + * + * As we don't strictly know what the swizzling is, it may be + * bit17 dependent, and so we need to also prevent the pages + * from being moved. + */ + dev_priv->quirks |= QUIRK_PIN_SWIZZLED_PAGES; + swizzle_x = I915_BIT_6_SWIZZLE_NONE; + swizzle_y = I915_BIT_6_SWIZZLE_NONE; + } + + dev_priv->mm.bit_6_swizzle_x = swizzle_x; + dev_priv->mm.bit_6_swizzle_y = swizzle_y; +} + +/* + * Swap every 64 bytes of this page around, to account for it having a new + * bit 17 of its physical address and therefore being interpreted differently + * by the GPU. + */ +static void +i915_gem_swizzle_page(struct page *page) +{ + char temp[64]; + char *vaddr; + int i; + + vaddr = kmap(page); + + for (i = 0; i < PAGE_SIZE; i += 128) { + memcpy(temp, &vaddr[i], 64); + memcpy(&vaddr[i], &vaddr[i + 64], 64); + memcpy(&vaddr[i + 64], temp, 64); + } + + kunmap(page); +} + +/** + * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling + * @obj: i915 GEM buffer object + * + * This function fixes up the swizzling in case any page frame number for this + * object has changed in bit 17 since that state has been saved with + * i915_gem_object_save_bit_17_swizzle(). + * + * This is called when pinning backing storage again, since the kernel is free + * to move unpinned backing storage around (either by directly moving pages or + * by swapping them out and back in again). + */ +void +i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) +{ + struct sg_page_iter sg_iter; + int i; + + if (obj->bit_17 == NULL) + return; + + i = 0; + for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { + struct page *page = sg_page_iter_page(&sg_iter); + char new_bit_17 = page_to_phys(page) >> 17; + if ((new_bit_17 & 0x1) != + (test_bit(i, obj->bit_17) != 0)) { + i915_gem_swizzle_page(page); + set_page_dirty(page); + } + i++; + } +} + +/** + * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling + * @obj: i915 GEM buffer object + * + * This function saves the bit 17 of each page frame number so that swizzling + * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must + * be called before the backing storage can be unpinned. + */ +void +i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) +{ + struct sg_page_iter sg_iter; + int page_count = obj->base.size >> PAGE_SHIFT; + int i; + + if (obj->bit_17 == NULL) { + obj->bit_17 = kcalloc(BITS_TO_LONGS(page_count), + sizeof(long), GFP_KERNEL); + if (obj->bit_17 == NULL) { + DRM_ERROR("Failed to allocate memory for bit 17 " + "record\n"); + return; + } + } + + i = 0; + for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { + if (page_to_phys(sg_page_iter_page(&sg_iter)) & (1 << 17)) + __set_bit(i, obj->bit_17); + else + __clear_bit(i, obj->bit_17); + i++; + } +} diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 31e8269e6e3d..1f7e6b9df45d 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -24,6 +24,7 @@ */ #include <linux/seq_file.h> +#include <linux/stop_machine.h> #include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" @@ -104,9 +105,11 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) { bool has_aliasing_ppgtt; bool has_full_ppgtt; + bool has_full_48bit_ppgtt; has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; + has_full_48bit_ppgtt = IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9; if (intel_vgpu_active(dev)) has_full_ppgtt = false; /* emulation is too hard */ @@ -125,6 +128,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) if (enable_ppgtt == 2 && has_full_ppgtt) return 2; + if (enable_ppgtt == 3 && has_full_48bit_ppgtt) + return 3; + #ifdef CONFIG_INTEL_IOMMU /* Disable ppgtt on SNB if VT-d is on. */ if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { @@ -141,7 +147,7 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) } if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) - return 2; + return has_full_48bit_ppgtt ? 3 : 2; else return has_aliasing_ppgtt ? 1 : 0; } @@ -192,9 +198,8 @@ static gen8_pte_t gen8_pte_encode(dma_addr_t addr, return pte; } -static gen8_pde_t gen8_pde_encode(struct drm_device *dev, - dma_addr_t addr, - enum i915_cache_level level) +static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, + const enum i915_cache_level level) { gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; pde |= addr; @@ -205,6 +210,9 @@ static gen8_pde_t gen8_pde_encode(struct drm_device *dev, return pde; } +#define gen8_pdpe_encode gen8_pde_encode +#define gen8_pml4e_encode gen8_pde_encode + static gen6_pte_t snb_pte_encode(dma_addr_t addr, enum i915_cache_level level, bool valid, u32 unused) @@ -301,75 +309,120 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr, return pte; } -#define i915_dma_unmap_single(px, dev) \ - __i915_dma_unmap_single((px)->daddr, dev) - -static void __i915_dma_unmap_single(dma_addr_t daddr, - struct drm_device *dev) +static int __setup_page_dma(struct drm_device *dev, + struct i915_page_dma *p, gfp_t flags) { struct device *device = &dev->pdev->dev; - dma_unmap_page(device, daddr, 4096, PCI_DMA_BIDIRECTIONAL); + p->page = alloc_page(flags); + if (!p->page) + return -ENOMEM; + + p->daddr = dma_map_page(device, + p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL); + + if (dma_mapping_error(device, p->daddr)) { + __free_page(p->page); + return -EINVAL; + } + + return 0; } -/** - * i915_dma_map_single() - Create a dma mapping for a page table/dir/etc. - * @px: Page table/dir/etc to get a DMA map for - * @dev: drm device - * - * Page table allocations are unified across all gens. They always require a - * single 4k allocation, as well as a DMA mapping. If we keep the structs - * symmetric here, the simple macro covers us for every page table type. - * - * Return: 0 if success. - */ -#define i915_dma_map_single(px, dev) \ - i915_dma_map_page_single((px)->page, (dev), &(px)->daddr) +static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) +{ + return __setup_page_dma(dev, p, GFP_KERNEL); +} -static int i915_dma_map_page_single(struct page *page, - struct drm_device *dev, - dma_addr_t *daddr) +static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) { - struct device *device = &dev->pdev->dev; + if (WARN_ON(!p->page)) + return; - *daddr = dma_map_page(device, page, 0, 4096, PCI_DMA_BIDIRECTIONAL); - if (dma_mapping_error(device, *daddr)) - return -ENOMEM; + dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL); + __free_page(p->page); + memset(p, 0, sizeof(*p)); +} - return 0; +static void *kmap_page_dma(struct i915_page_dma *p) +{ + return kmap_atomic(p->page); } -static void unmap_and_free_pt(struct i915_page_table *pt, - struct drm_device *dev) +/* We use the flushing unmap only with ppgtt structures: + * page directories, page tables and scratch pages. + */ +static void kunmap_page_dma(struct drm_device *dev, void *vaddr) { - if (WARN_ON(!pt->page)) - return; + /* There are only few exceptions for gen >=6. chv and bxt. + * And we are not sure about the latter so play safe for now. + */ + if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) + drm_clflush_virt_range(vaddr, PAGE_SIZE); - i915_dma_unmap_single(pt, dev); - __free_page(pt->page); - kfree(pt->used_ptes); - kfree(pt); + kunmap_atomic(vaddr); } -static void gen8_initialize_pt(struct i915_address_space *vm, - struct i915_page_table *pt) +#define kmap_px(px) kmap_page_dma(px_base(px)) +#define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) + +#define setup_px(dev, px) setup_page_dma((dev), px_base(px)) +#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) +#define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) +#define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) + +static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, + const uint64_t val) { - gen8_pte_t *pt_vaddr, scratch_pte; int i; + uint64_t * const vaddr = kmap_page_dma(p); - pt_vaddr = kmap_atomic(pt->page); - scratch_pte = gen8_pte_encode(vm->scratch.addr, - I915_CACHE_LLC, true); + for (i = 0; i < 512; i++) + vaddr[i] = val; + + kunmap_page_dma(dev, vaddr); +} + +static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, + const uint32_t val32) +{ + uint64_t v = val32; + + v = v << 32 | val32; + + fill_page_dma(dev, p, v); +} + +static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) +{ + struct i915_page_scratch *sp; + int ret; - for (i = 0; i < GEN8_PTES; i++) - pt_vaddr[i] = scratch_pte; + sp = kzalloc(sizeof(*sp), GFP_KERNEL); + if (sp == NULL) + return ERR_PTR(-ENOMEM); + + ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); + if (ret) { + kfree(sp); + return ERR_PTR(ret); + } - if (!HAS_LLC(vm->dev)) - drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); - kunmap_atomic(pt_vaddr); + set_pages_uc(px_page(sp), 1); + + return sp; } -static struct i915_page_table *alloc_pt_single(struct drm_device *dev) +static void free_scratch_page(struct drm_device *dev, + struct i915_page_scratch *sp) +{ + set_pages_wb(px_page(sp), 1); + + cleanup_px(dev, sp); + kfree(sp); +} + +static struct i915_page_table *alloc_pt(struct drm_device *dev) { struct i915_page_table *pt; const size_t count = INTEL_INFO(dev)->gen >= 8 ? @@ -386,19 +439,13 @@ static struct i915_page_table *alloc_pt_single(struct drm_device *dev) if (!pt->used_ptes) goto fail_bitmap; - pt->page = alloc_page(GFP_KERNEL); - if (!pt->page) - goto fail_page; - - ret = i915_dma_map_single(pt, dev); + ret = setup_px(dev, pt); if (ret) - goto fail_dma; + goto fail_page_m; return pt; -fail_dma: - __free_page(pt->page); -fail_page: +fail_page_m: kfree(pt->used_ptes); fail_bitmap: kfree(pt); @@ -406,18 +453,38 @@ fail_bitmap: return ERR_PTR(ret); } -static void unmap_and_free_pd(struct i915_page_directory *pd, - struct drm_device *dev) +static void free_pt(struct drm_device *dev, struct i915_page_table *pt) { - if (pd->page) { - i915_dma_unmap_single(pd, dev); - __free_page(pd->page); - kfree(pd->used_pdes); - kfree(pd); - } + cleanup_px(dev, pt); + kfree(pt->used_ptes); + kfree(pt); +} + +static void gen8_initialize_pt(struct i915_address_space *vm, + struct i915_page_table *pt) +{ + gen8_pte_t scratch_pte; + + scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, true); + + fill_px(vm->dev, pt, scratch_pte); +} + +static void gen6_initialize_pt(struct i915_address_space *vm, + struct i915_page_table *pt) +{ + gen6_pte_t scratch_pte; + + WARN_ON(px_dma(vm->scratch_page) == 0); + + scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, true, 0); + + fill32_px(vm->dev, pt, scratch_pte); } -static struct i915_page_directory *alloc_pd_single(struct drm_device *dev) +static struct i915_page_directory *alloc_pd(struct drm_device *dev) { struct i915_page_directory *pd; int ret = -ENOMEM; @@ -429,63 +496,196 @@ static struct i915_page_directory *alloc_pd_single(struct drm_device *dev) pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), sizeof(*pd->used_pdes), GFP_KERNEL); if (!pd->used_pdes) - goto free_pd; - - pd->page = alloc_page(GFP_KERNEL); - if (!pd->page) - goto free_bitmap; + goto fail_bitmap; - ret = i915_dma_map_single(pd, dev); + ret = setup_px(dev, pd); if (ret) - goto free_page; + goto fail_page_m; return pd; -free_page: - __free_page(pd->page); -free_bitmap: +fail_page_m: kfree(pd->used_pdes); -free_pd: +fail_bitmap: kfree(pd); return ERR_PTR(ret); } +static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) +{ + if (px_page(pd)) { + cleanup_px(dev, pd); + kfree(pd->used_pdes); + kfree(pd); + } +} + +static void gen8_initialize_pd(struct i915_address_space *vm, + struct i915_page_directory *pd) +{ + gen8_pde_t scratch_pde; + + scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); + + fill_px(vm->dev, pd, scratch_pde); +} + +static int __pdp_init(struct drm_device *dev, + struct i915_page_directory_pointer *pdp) +{ + size_t pdpes = I915_PDPES_PER_PDP(dev); + + pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), + sizeof(unsigned long), + GFP_KERNEL); + if (!pdp->used_pdpes) + return -ENOMEM; + + pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), + GFP_KERNEL); + if (!pdp->page_directory) { + kfree(pdp->used_pdpes); + /* the PDP might be the statically allocated top level. Keep it + * as clean as possible */ + pdp->used_pdpes = NULL; + return -ENOMEM; + } + + return 0; +} + +static void __pdp_fini(struct i915_page_directory_pointer *pdp) +{ + kfree(pdp->used_pdpes); + kfree(pdp->page_directory); + pdp->page_directory = NULL; +} + +static struct +i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) +{ + struct i915_page_directory_pointer *pdp; + int ret = -ENOMEM; + + WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); + + pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); + if (!pdp) + return ERR_PTR(-ENOMEM); + + ret = __pdp_init(dev, pdp); + if (ret) + goto fail_bitmap; + + ret = setup_px(dev, pdp); + if (ret) + goto fail_page_m; + + return pdp; + +fail_page_m: + __pdp_fini(pdp); +fail_bitmap: + kfree(pdp); + + return ERR_PTR(ret); +} + +static void free_pdp(struct drm_device *dev, + struct i915_page_directory_pointer *pdp) +{ + __pdp_fini(pdp); + if (USES_FULL_48BIT_PPGTT(dev)) { + cleanup_px(dev, pdp); + kfree(pdp); + } +} + +static void gen8_initialize_pdp(struct i915_address_space *vm, + struct i915_page_directory_pointer *pdp) +{ + gen8_ppgtt_pdpe_t scratch_pdpe; + + scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); + + fill_px(vm->dev, pdp, scratch_pdpe); +} + +static void gen8_initialize_pml4(struct i915_address_space *vm, + struct i915_pml4 *pml4) +{ + gen8_ppgtt_pml4e_t scratch_pml4e; + + scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), + I915_CACHE_LLC); + + fill_px(vm->dev, pml4, scratch_pml4e); +} + +static void +gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, + struct i915_page_directory_pointer *pdp, + struct i915_page_directory *pd, + int index) +{ + gen8_ppgtt_pdpe_t *page_directorypo; + + if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) + return; + + page_directorypo = kmap_px(pdp); + page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); + kunmap_px(ppgtt, page_directorypo); +} + +static void +gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, + struct i915_pml4 *pml4, + struct i915_page_directory_pointer *pdp, + int index) +{ + gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); + + WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); + pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); + kunmap_px(ppgtt, pagemap); +} + /* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp(struct intel_engine_cs *ring, +static int gen8_write_pdp(struct drm_i915_gem_request *req, unsigned entry, dma_addr_t addr) { + struct intel_engine_cs *ring = req->ring; int ret; BUG_ON(entry >= 4); - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); + intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(ring, entry)); intel_ring_emit(ring, upper_32_bits(addr)); intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); + intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(ring, entry)); intel_ring_emit(ring, lower_32_bits(addr)); intel_ring_advance(ring); return 0; } -static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring) +static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt, + struct drm_i915_gem_request *req) { int i, ret; for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { - struct i915_page_directory *pd = ppgtt->pdp.page_directory[i]; - dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr; - /* The page directory might be NULL, but we need to clear out - * whatever the previous context might have used. */ - ret = gen8_write_pdp(ring, i, pd_daddr); + const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); + + ret = gen8_write_pdp(req, i, pd_daddr); if (ret) return ret; } @@ -493,184 +693,313 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, return 0; } -static void gen8_ppgtt_clear_range(struct i915_address_space *vm, - uint64_t start, - uint64_t length, - bool use_scratch) +static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, + struct drm_i915_gem_request *req) +{ + return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); +} + +static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, + struct i915_page_directory_pointer *pdp, + uint64_t start, + uint64_t length, + gen8_pte_t scratch_pte) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); - gen8_pte_t *pt_vaddr, scratch_pte; - unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; - unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; - unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; + gen8_pte_t *pt_vaddr; + unsigned pdpe = gen8_pdpe_index(start); + unsigned pde = gen8_pde_index(start); + unsigned pte = gen8_pte_index(start); unsigned num_entries = length >> PAGE_SHIFT; unsigned last_pte, i; - scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr, - I915_CACHE_LLC, use_scratch); + if (WARN_ON(!pdp)) + return; while (num_entries) { struct i915_page_directory *pd; struct i915_page_table *pt; - struct page *page_table; - if (WARN_ON(!ppgtt->pdp.page_directory[pdpe])) + if (WARN_ON(!pdp->page_directory[pdpe])) break; - pd = ppgtt->pdp.page_directory[pdpe]; + pd = pdp->page_directory[pdpe]; if (WARN_ON(!pd->page_table[pde])) break; pt = pd->page_table[pde]; - if (WARN_ON(!pt->page)) + if (WARN_ON(!px_page(pt))) break; - page_table = pt->page; - last_pte = pte + num_entries; if (last_pte > GEN8_PTES) last_pte = GEN8_PTES; - pt_vaddr = kmap_atomic(page_table); + pt_vaddr = kmap_px(pt); for (i = pte; i < last_pte; i++) { pt_vaddr[i] = scratch_pte; num_entries--; } - if (!HAS_LLC(ppgtt->base.dev)) - drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); - kunmap_atomic(pt_vaddr); + kunmap_px(ppgtt, pt); pte = 0; if (++pde == I915_PDES) { - pdpe++; + if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) + break; pde = 0; } } } -static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, - struct sg_table *pages, - uint64_t start, - enum i915_cache_level cache_level, u32 unused) +static void gen8_ppgtt_clear_range(struct i915_address_space *vm, + uint64_t start, + uint64_t length, + bool use_scratch) +{ + struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, use_scratch); + + if (!USES_FULL_48BIT_PPGTT(vm->dev)) { + gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, + scratch_pte); + } else { + uint64_t templ4, pml4e; + struct i915_page_directory_pointer *pdp; + + gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { + gen8_ppgtt_clear_pte_range(vm, pdp, start, length, + scratch_pte); + } + } +} + +static void +gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, + struct i915_page_directory_pointer *pdp, + struct sg_page_iter *sg_iter, + uint64_t start, + enum i915_cache_level cache_level) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); gen8_pte_t *pt_vaddr; - unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; - unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; - unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; - struct sg_page_iter sg_iter; + unsigned pdpe = gen8_pdpe_index(start); + unsigned pde = gen8_pde_index(start); + unsigned pte = gen8_pte_index(start); pt_vaddr = NULL; - for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { - if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES)) - break; - + while (__sg_page_iter_next(sg_iter)) { if (pt_vaddr == NULL) { - struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe]; + struct i915_page_directory *pd = pdp->page_directory[pdpe]; struct i915_page_table *pt = pd->page_table[pde]; - struct page *page_table = pt->page; - - pt_vaddr = kmap_atomic(page_table); + pt_vaddr = kmap_px(pt); } pt_vaddr[pte] = - gen8_pte_encode(sg_page_iter_dma_address(&sg_iter), + gen8_pte_encode(sg_page_iter_dma_address(sg_iter), cache_level, true); if (++pte == GEN8_PTES) { - if (!HAS_LLC(ppgtt->base.dev)) - drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); - kunmap_atomic(pt_vaddr); + kunmap_px(ppgtt, pt_vaddr); pt_vaddr = NULL; if (++pde == I915_PDES) { - pdpe++; + if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) + break; pde = 0; } pte = 0; } } - if (pt_vaddr) { - if (!HAS_LLC(ppgtt->base.dev)) - drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); - kunmap_atomic(pt_vaddr); - } -} -static void __gen8_do_map_pt(gen8_pde_t * const pde, - struct i915_page_table *pt, - struct drm_device *dev) -{ - gen8_pde_t entry = - gen8_pde_encode(dev, pt->daddr, I915_CACHE_LLC); - *pde = entry; + if (pt_vaddr) + kunmap_px(ppgtt, pt_vaddr); } -static void gen8_initialize_pd(struct i915_address_space *vm, - struct i915_page_directory *pd) +static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, + struct sg_table *pages, + uint64_t start, + enum i915_cache_level cache_level, + u32 unused) { struct i915_hw_ppgtt *ppgtt = - container_of(vm, struct i915_hw_ppgtt, base); - gen8_pde_t *page_directory; - struct i915_page_table *pt; - int i; + container_of(vm, struct i915_hw_ppgtt, base); + struct sg_page_iter sg_iter; - page_directory = kmap_atomic(pd->page); - pt = ppgtt->scratch_pt; - for (i = 0; i < I915_PDES; i++) - /* Map the PDE to the page table */ - __gen8_do_map_pt(page_directory + i, pt, vm->dev); + __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); - if (!HAS_LLC(vm->dev)) - drm_clflush_virt_range(page_directory, PAGE_SIZE); - kunmap_atomic(page_directory); + if (!USES_FULL_48BIT_PPGTT(vm->dev)) { + gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, + cache_level); + } else { + struct i915_page_directory_pointer *pdp; + uint64_t templ4, pml4e; + uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT; + + gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { + gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, + start, cache_level); + } + } } -static void gen8_free_page_tables(struct i915_page_directory *pd, struct drm_device *dev) +static void gen8_free_page_tables(struct drm_device *dev, + struct i915_page_directory *pd) { int i; - if (!pd->page) + if (!px_page(pd)) return; for_each_set_bit(i, pd->used_pdes, I915_PDES) { if (WARN_ON(!pd->page_table[i])) continue; - unmap_and_free_pt(pd->page_table[i], dev); + free_pt(dev, pd->page_table[i]); pd->page_table[i] = NULL; } } -static void gen8_ppgtt_cleanup(struct i915_address_space *vm) +static int gen8_init_scratch(struct i915_address_space *vm) { - struct i915_hw_ppgtt *ppgtt = - container_of(vm, struct i915_hw_ppgtt, base); + struct drm_device *dev = vm->dev; + + vm->scratch_page = alloc_scratch_page(dev); + if (IS_ERR(vm->scratch_page)) + return PTR_ERR(vm->scratch_page); + + vm->scratch_pt = alloc_pt(dev); + if (IS_ERR(vm->scratch_pt)) { + free_scratch_page(dev, vm->scratch_page); + return PTR_ERR(vm->scratch_pt); + } + + vm->scratch_pd = alloc_pd(dev); + if (IS_ERR(vm->scratch_pd)) { + free_pt(dev, vm->scratch_pt); + free_scratch_page(dev, vm->scratch_page); + return PTR_ERR(vm->scratch_pd); + } + + if (USES_FULL_48BIT_PPGTT(dev)) { + vm->scratch_pdp = alloc_pdp(dev); + if (IS_ERR(vm->scratch_pdp)) { + free_pd(dev, vm->scratch_pd); + free_pt(dev, vm->scratch_pt); + free_scratch_page(dev, vm->scratch_page); + return PTR_ERR(vm->scratch_pdp); + } + } + + gen8_initialize_pt(vm, vm->scratch_pt); + gen8_initialize_pd(vm, vm->scratch_pd); + if (USES_FULL_48BIT_PPGTT(dev)) + gen8_initialize_pdp(vm, vm->scratch_pdp); + + return 0; +} + +static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) +{ + enum vgt_g2v_type msg; + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; int i; - for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) { - if (WARN_ON(!ppgtt->pdp.page_directory[i])) + if (USES_FULL_48BIT_PPGTT(dev)) { + u64 daddr = px_dma(&ppgtt->pml4); + + I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr)); + I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr)); + + msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : + VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); + } else { + for (i = 0; i < GEN8_LEGACY_PDPES; i++) { + u64 daddr = i915_page_dir_dma_addr(ppgtt, i); + + I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr)); + I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr)); + } + + msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : + VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); + } + + I915_WRITE(vgtif_reg(g2v_notify), msg); + + return 0; +} + +static void gen8_free_scratch(struct i915_address_space *vm) +{ + struct drm_device *dev = vm->dev; + + if (USES_FULL_48BIT_PPGTT(dev)) + free_pdp(dev, vm->scratch_pdp); + free_pd(dev, vm->scratch_pd); + free_pt(dev, vm->scratch_pt); + free_scratch_page(dev, vm->scratch_page); +} + +static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, + struct i915_page_directory_pointer *pdp) +{ + int i; + + for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { + if (WARN_ON(!pdp->page_directory[i])) continue; - gen8_free_page_tables(ppgtt->pdp.page_directory[i], ppgtt->base.dev); - unmap_and_free_pd(ppgtt->pdp.page_directory[i], ppgtt->base.dev); + gen8_free_page_tables(dev, pdp->page_directory[i]); + free_pd(dev, pdp->page_directory[i]); } - unmap_and_free_pd(ppgtt->scratch_pd, ppgtt->base.dev); - unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev); + free_pdp(dev, pdp); +} + +static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) +{ + int i; + + for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { + if (WARN_ON(!ppgtt->pml4.pdps[i])) + continue; + + gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); + } + + cleanup_px(ppgtt->base.dev, &ppgtt->pml4); +} + +static void gen8_ppgtt_cleanup(struct i915_address_space *vm) +{ + struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + + if (intel_vgpu_active(vm->dev)) + gen8_ppgtt_notify_vgt(ppgtt, false); + + if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) + gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); + else + gen8_ppgtt_cleanup_4lvl(ppgtt); + + gen8_free_scratch(vm); } /** * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. - * @ppgtt: Master ppgtt structure. - * @pd: Page directory for this address range. + * @vm: Master vm structure. + * @pd: Page directory for this address range. * @start: Starting virtual address to begin allocations. - * @length Size of the allocations. + * @length: Size of the allocations. * @new_pts: Bitmap set by function with new allocations. Likely used by the * caller to free on error. * @@ -683,50 +1012,51 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm) * * Return: 0 if success; negative error code otherwise. */ -static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt, +static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, struct i915_page_directory *pd, uint64_t start, uint64_t length, unsigned long *new_pts) { - struct drm_device *dev = ppgtt->base.dev; + struct drm_device *dev = vm->dev; struct i915_page_table *pt; uint64_t temp; uint32_t pde; gen8_for_each_pde(pt, pd, start, length, temp, pde) { /* Don't reallocate page tables */ - if (pt) { + if (test_bit(pde, pd->used_pdes)) { /* Scratch is never allocated this way */ - WARN_ON(pt == ppgtt->scratch_pt); + WARN_ON(pt == vm->scratch_pt); continue; } - pt = alloc_pt_single(dev); + pt = alloc_pt(dev); if (IS_ERR(pt)) goto unwind_out; - gen8_initialize_pt(&ppgtt->base, pt); + gen8_initialize_pt(vm, pt); pd->page_table[pde] = pt; - set_bit(pde, new_pts); + __set_bit(pde, new_pts); + trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); } return 0; unwind_out: for_each_set_bit(pde, new_pts, I915_PDES) - unmap_and_free_pt(pd->page_table[pde], dev); + free_pt(dev, pd->page_table[pde]); return -ENOMEM; } /** * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. - * @ppgtt: Master ppgtt structure. + * @vm: Master vm structure. * @pdp: Page directory pointer for this address range. * @start: Starting virtual address to begin allocations. - * @length Size of the allocations. - * @new_pds Bitmap set by function with new allocations. Likely used by the + * @length: Size of the allocations. + * @new_pds: Bitmap set by function with new allocations. Likely used by the * caller to free on error. * * Allocate the required number of page directories starting at the pde index of @@ -743,51 +1073,102 @@ unwind_out: * * Return: 0 if success; negative error code otherwise. */ -static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt, - struct i915_page_directory_pointer *pdp, - uint64_t start, - uint64_t length, - unsigned long *new_pds) +static int +gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, + struct i915_page_directory_pointer *pdp, + uint64_t start, + uint64_t length, + unsigned long *new_pds) { - struct drm_device *dev = ppgtt->base.dev; + struct drm_device *dev = vm->dev; struct i915_page_directory *pd; uint64_t temp; uint32_t pdpe; + uint32_t pdpes = I915_PDPES_PER_PDP(dev); - WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES)); - - /* FIXME: upper bound must not overflow 32 bits */ - WARN_ON((start + length) > (1ULL << 32)); + WARN_ON(!bitmap_empty(new_pds, pdpes)); gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { - if (pd) + if (test_bit(pdpe, pdp->used_pdpes)) continue; - pd = alloc_pd_single(dev); + pd = alloc_pd(dev); if (IS_ERR(pd)) goto unwind_out; - gen8_initialize_pd(&ppgtt->base, pd); + gen8_initialize_pd(vm, pd); pdp->page_directory[pdpe] = pd; - set_bit(pdpe, new_pds); + __set_bit(pdpe, new_pds); + trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); } return 0; unwind_out: - for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES) - unmap_and_free_pd(pdp->page_directory[pdpe], dev); + for_each_set_bit(pdpe, new_pds, pdpes) + free_pd(dev, pdp->page_directory[pdpe]); return -ENOMEM; } -static void -free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts) +/** + * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range. + * @vm: Master vm structure. + * @pml4: Page map level 4 for this address range. + * @start: Starting virtual address to begin allocations. + * @length: Size of the allocations. + * @new_pdps: Bitmap set by function with new allocations. Likely used by the + * caller to free on error. + * + * Allocate the required number of page directory pointers. Extremely similar to + * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs(). + * The main difference is here we are limited by the pml4 boundary (instead of + * the page directory pointer). + * + * Return: 0 if success; negative error code otherwise. + */ +static int +gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, + struct i915_pml4 *pml4, + uint64_t start, + uint64_t length, + unsigned long *new_pdps) { - int i; + struct drm_device *dev = vm->dev; + struct i915_page_directory_pointer *pdp; + uint64_t temp; + uint32_t pml4e; + + WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4)); + + gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { + if (!test_bit(pml4e, pml4->used_pml4es)) { + pdp = alloc_pdp(dev); + if (IS_ERR(pdp)) + goto unwind_out; + + gen8_initialize_pdp(vm, pdp); + pml4->pdps[pml4e] = pdp; + __set_bit(pml4e, new_pdps); + trace_i915_page_directory_pointer_entry_alloc(vm, + pml4e, + start, + GEN8_PML4E_SHIFT); + } + } + + return 0; + +unwind_out: + for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) + free_pdp(dev, pml4->pdps[pml4e]); - for (i = 0; i < GEN8_LEGACY_PDPES; i++) - kfree(new_pts[i]); + return -ENOMEM; +} + +static void +free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts) +{ kfree(new_pts); kfree(new_pds); } @@ -797,28 +1178,20 @@ free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts) */ static int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, - unsigned long ***new_pts) + unsigned long **new_pts, + uint32_t pdpes) { - int i; unsigned long *pds; - unsigned long **pts; + unsigned long *pts; - pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL); + pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY); if (!pds) return -ENOMEM; - pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL); - if (!pts) { - kfree(pds); - return -ENOMEM; - } - - for (i = 0; i < GEN8_LEGACY_PDPES; i++) { - pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES), - sizeof(unsigned long), GFP_KERNEL); - if (!pts[i]) - goto err_out; - } + pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long), + GFP_TEMPORARY); + if (!pts) + goto err_out; *new_pds = pds; *new_pts = pts; @@ -830,42 +1203,58 @@ err_out: return -ENOMEM; } -static int gen8_alloc_va_range(struct i915_address_space *vm, - uint64_t start, - uint64_t length) +/* PDE TLBs are a pain to invalidate on GEN8+. When we modify + * the page table structures, we mark them dirty so that + * context switching/execlist queuing code takes extra steps + * to ensure that tlbs are flushed. + */ +static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) +{ + ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; +} + +static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, + struct i915_page_directory_pointer *pdp, + uint64_t start, + uint64_t length) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); - unsigned long *new_page_dirs, **new_page_tables; + unsigned long *new_page_dirs, *new_page_tables; + struct drm_device *dev = vm->dev; struct i915_page_directory *pd; const uint64_t orig_start = start; const uint64_t orig_length = length; uint64_t temp; uint32_t pdpe; + uint32_t pdpes = I915_PDPES_PER_PDP(dev); int ret; /* Wrap is never okay since we can only represent 48b, and we don't * actually use the other side of the canonical address space. */ if (WARN_ON(start + length < start)) - return -ERANGE; + return -ENODEV; + + if (WARN_ON(start + length > vm->total)) + return -ENODEV; - ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables); + ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); if (ret) return ret; /* Do the allocations first so we can easily bail out */ - ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length, - new_page_dirs); + ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, + new_page_dirs); if (ret) { free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); return ret; } /* For every page directory referenced, allocate page tables */ - gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret = gen8_ppgtt_alloc_pagetabs(ppgtt, pd, start, length, - new_page_tables[pdpe]); + gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { + ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, + new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES)); if (ret) goto err_out; } @@ -875,10 +1264,10 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, /* Allocations have completed successfully, so set the bitmaps, and do * the mappings. */ - gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) { - gen8_pde_t *const page_directory = kmap_atomic(pd->page); + gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { + gen8_pde_t *const page_directory = kmap_px(pd); struct i915_page_table *pt; - uint64_t pd_len = gen8_clamp_pd(start, length); + uint64_t pd_len = length; uint64_t pd_start = start; uint32_t pde; @@ -897,36 +1286,213 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, gen8_pte_count(pd_start, pd_len)); /* Our pde is now pointing to the pagetable, pt */ - set_bit(pde, pd->used_pdes); + __set_bit(pde, pd->used_pdes); /* Map the PDE to the page table */ - __gen8_do_map_pt(page_directory + pde, pt, vm->dev); + page_directory[pde] = gen8_pde_encode(px_dma(pt), + I915_CACHE_LLC); + trace_i915_page_table_entry_map(&ppgtt->base, pde, pt, + gen8_pte_index(start), + gen8_pte_count(start, length), + GEN8_PTES); /* NB: We haven't yet mapped ptes to pages. At this * point we're still relying on insert_entries() */ } - if (!HAS_LLC(vm->dev)) - drm_clflush_virt_range(page_directory, PAGE_SIZE); + kunmap_px(ppgtt, page_directory); + __set_bit(pdpe, pdp->used_pdpes); + gen8_setup_page_directory(ppgtt, pdp, pd, pdpe); + } - kunmap_atomic(page_directory); + free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); + mark_tlbs_dirty(ppgtt); + return 0; - set_bit(pdpe, ppgtt->pdp.used_pdpes); +err_out: + while (pdpe--) { + for_each_set_bit(temp, new_page_tables + pdpe * + BITS_TO_LONGS(I915_PDES), I915_PDES) + free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); } + for_each_set_bit(pdpe, new_page_dirs, pdpes) + free_pd(dev, pdp->page_directory[pdpe]); + free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); + mark_tlbs_dirty(ppgtt); + return ret; +} + +static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, + struct i915_pml4 *pml4, + uint64_t start, + uint64_t length) +{ + DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4); + struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + struct i915_page_directory_pointer *pdp; + uint64_t temp, pml4e; + int ret = 0; + + /* Do the pml4 allocations first, so we don't need to track the newly + * allocated tables below the pdp */ + bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4); + + /* The pagedirectory and pagetable allocations are done in the shared 3 + * and 4 level code. Just allocate the pdps. + */ + ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, + new_pdps); + if (ret) + return ret; + + WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2, + "The allocation has spanned more than 512GB. " + "It is highly likely this is incorrect."); + + gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { + WARN_ON(!pdp); + + ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); + if (ret) + goto err_out; + + gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e); + } + + bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es, + GEN8_PML4ES_PER_PML4); + return 0; err_out: - while (pdpe--) { - for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES) - unmap_and_free_pt(ppgtt->pdp.page_directory[pdpe]->page_table[temp], vm->dev); + for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) + gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); + + return ret; +} + +static int gen8_alloc_va_range(struct i915_address_space *vm, + uint64_t start, uint64_t length) +{ + struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + + if (USES_FULL_48BIT_PPGTT(vm->dev)) + return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); + else + return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); +} + +static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp, + uint64_t start, uint64_t length, + gen8_pte_t scratch_pte, + struct seq_file *m) +{ + struct i915_page_directory *pd; + uint64_t temp; + uint32_t pdpe; + + gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { + struct i915_page_table *pt; + uint64_t pd_len = length; + uint64_t pd_start = start; + uint32_t pde; + + if (!test_bit(pdpe, pdp->used_pdpes)) + continue; + + seq_printf(m, "\tPDPE #%d\n", pdpe); + gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { + uint32_t pte; + gen8_pte_t *pt_vaddr; + + if (!test_bit(pde, pd->used_pdes)) + continue; + + pt_vaddr = kmap_px(pt); + for (pte = 0; pte < GEN8_PTES; pte += 4) { + uint64_t va = + (pdpe << GEN8_PDPE_SHIFT) | + (pde << GEN8_PDE_SHIFT) | + (pte << GEN8_PTE_SHIFT); + int i; + bool found = false; + + for (i = 0; i < 4; i++) + if (pt_vaddr[pte + i] != scratch_pte) + found = true; + if (!found) + continue; + + seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); + for (i = 0; i < 4; i++) { + if (pt_vaddr[pte + i] != scratch_pte) + seq_printf(m, " %llx", pt_vaddr[pte + i]); + else + seq_puts(m, " SCRATCH "); + } + seq_puts(m, "\n"); + } + /* don't use kunmap_px, it could trigger + * an unnecessary flush. + */ + kunmap_atomic(pt_vaddr); + } } +} + +static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) +{ + struct i915_address_space *vm = &ppgtt->base; + uint64_t start = ppgtt->base.start; + uint64_t length = ppgtt->base.total; + gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, true); + + if (!USES_FULL_48BIT_PPGTT(vm->dev)) { + gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); + } else { + uint64_t templ4, pml4e; + struct i915_pml4 *pml4 = &ppgtt->pml4; + struct i915_page_directory_pointer *pdp; - for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES) - unmap_and_free_pd(ppgtt->pdp.page_directory[pdpe], vm->dev); + gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) { + if (!test_bit(pml4e, pml4->used_pml4es)) + continue; + + seq_printf(m, " PML4E #%llu\n", pml4e); + gen8_dump_pdp(pdp, start, length, scratch_pte, m); + } + } +} + +static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) +{ + unsigned long *new_page_dirs, *new_page_tables; + uint32_t pdpes = I915_PDPES_PER_PDP(dev); + int ret; + + /* We allocate temp bitmap for page tables for no gain + * but as this is for init only, lets keep the things simple + */ + ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); + if (ret) + return ret; + + /* Allocate for all pdps regardless of how the ppgtt + * was defined. + */ + ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp, + 0, 1ULL << 32, + new_page_dirs); + if (!ret) + *ppgtt->pdp.used_pdpes = *new_page_dirs; free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); + return ret; } @@ -939,36 +1505,56 @@ err_out: */ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) { - ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev); - if (IS_ERR(ppgtt->scratch_pt)) - return PTR_ERR(ppgtt->scratch_pt); - - ppgtt->scratch_pd = alloc_pd_single(ppgtt->base.dev); - if (IS_ERR(ppgtt->scratch_pd)) - return PTR_ERR(ppgtt->scratch_pd); + int ret; - gen8_initialize_pt(&ppgtt->base, ppgtt->scratch_pt); - gen8_initialize_pd(&ppgtt->base, ppgtt->scratch_pd); + ret = gen8_init_scratch(&ppgtt->base); + if (ret) + return ret; ppgtt->base.start = 0; - ppgtt->base.total = 1ULL << 32; - if (IS_ENABLED(CONFIG_X86_32)) - /* While we have a proliferation of size_t variables - * we cannot represent the full ppgtt size on 32bit, - * so limit it to the same size as the GGTT (currently - * 2GiB). - */ - ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total; ppgtt->base.cleanup = gen8_ppgtt_cleanup; ppgtt->base.allocate_va_range = gen8_alloc_va_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.unbind_vma = ppgtt_unbind_vma; ppgtt->base.bind_vma = ppgtt_bind_vma; + ppgtt->debug_dump = gen8_dump_ppgtt; + + if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { + ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); + if (ret) + goto free_scratch; + + gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); - ppgtt->switch_mm = gen8_mm_switch; + ppgtt->base.total = 1ULL << 48; + ppgtt->switch_mm = gen8_48b_mm_switch; + } else { + ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); + if (ret) + goto free_scratch; + + ppgtt->base.total = 1ULL << 32; + ppgtt->switch_mm = gen8_legacy_mm_switch; + trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base, + 0, 0, + GEN8_PML4E_SHIFT); + + if (intel_vgpu_active(ppgtt->base.dev)) { + ret = gen8_preallocate_top_level_pdps(ppgtt); + if (ret) + goto free_scratch; + } + } + + if (intel_vgpu_active(ppgtt->base.dev)) + gen8_ppgtt_notify_vgt(ppgtt, true); return 0; + +free_scratch: + gen8_free_scratch(&ppgtt->base); + return ret; } static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) @@ -980,12 +1566,13 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) uint32_t pte, pde, temp; uint32_t start = ppgtt->base.start, length = ppgtt->base.total; - scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0); + scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, true, 0); gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { u32 expected; gen6_pte_t *pt_vaddr; - dma_addr_t pt_addr = ppgtt->pd.page_table[pde]->daddr; + const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); pd_entry = readl(ppgtt->pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); @@ -996,7 +1583,8 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected); seq_printf(m, "\tPDE: %x\n", pd_entry); - pt_vaddr = kmap_atomic(ppgtt->pd.page_table[pde]->page); + pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]); + for (pte = 0; pte < GEN6_PTES; pte+=4) { unsigned long va = (pde * PAGE_SIZE * GEN6_PTES) + @@ -1018,7 +1606,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } seq_puts(m, "\n"); } - kunmap_atomic(pt_vaddr); + kunmap_px(ppgtt, pt_vaddr); } } @@ -1031,7 +1619,7 @@ static void gen6_write_pde(struct i915_page_directory *pd, container_of(pd, struct i915_hw_ppgtt, pd); u32 pd_entry; - pd_entry = GEN6_PDE_ADDR_ENCODE(pt->daddr); + pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt)); pd_entry |= GEN6_PDE_VALID; writel(pd_entry, ppgtt->pd_addr + pde); @@ -1056,29 +1644,30 @@ static void gen6_write_page_range(struct drm_i915_private *dev_priv, static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) { - BUG_ON(ppgtt->pd.pd_offset & 0x3f); + BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f); - return (ppgtt->pd.pd_offset / 64) << 16; + return (ppgtt->pd.base.ggtt_offset / 64) << 16; } static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int ret; /* NB: TLBs must be flushed and invalidated before a switch */ - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret) return ret; - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); + intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring)); intel_ring_emit(ring, PP_DIR_DCLV_2G); - intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); + intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring)); intel_ring_emit(ring, get_pd_offset(ppgtt)); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); @@ -1087,8 +1676,9 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, } static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); @@ -1097,30 +1687,31 @@ static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, } static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int ret; /* NB: TLBs must be flushed and invalidated before a switch */ - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret) return ret; - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); + intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring)); intel_ring_emit(ring, PP_DIR_DCLV_2G); - intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); + intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring)); intel_ring_emit(ring, get_pd_offset(ppgtt)); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); /* XXX: RCS is the only one to auto invalidate the TLBs? */ if (ring->id != RCS) { - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret) return ret; } @@ -1129,8 +1720,9 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, } static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring) + struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1150,8 +1742,9 @@ static void gen8_ppgtt_enable(struct drm_device *dev) int j; for_each_ring(ring, dev_priv, j) { + u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; I915_WRITE(RING_MODE_GEN7(ring), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); + _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); } } @@ -1214,19 +1807,20 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, unsigned first_pte = first_entry % GEN6_PTES; unsigned last_pte, i; - scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0); + scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, true, 0); while (num_entries) { last_pte = first_pte + num_entries; if (last_pte > GEN6_PTES) last_pte = GEN6_PTES; - pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page); + pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte; - kunmap_atomic(pt_vaddr); + kunmap_px(ppgtt, pt_vaddr); num_entries -= last_pte - first_pte; first_pte = 0; @@ -1250,54 +1844,25 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr = NULL; for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { if (pt_vaddr == NULL) - pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page); + pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter), cache_level, true, flags); if (++act_pte == GEN6_PTES) { - kunmap_atomic(pt_vaddr); + kunmap_px(ppgtt, pt_vaddr); pt_vaddr = NULL; act_pt++; act_pte = 0; } } if (pt_vaddr) - kunmap_atomic(pt_vaddr); -} - -/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we - * are switching between contexts with the same LRCA, we also must do a force - * restore. - */ -static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) -{ - /* If current vm != vm, */ - ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; -} - -static void gen6_initialize_pt(struct i915_address_space *vm, - struct i915_page_table *pt) -{ - gen6_pte_t *pt_vaddr, scratch_pte; - int i; - - WARN_ON(vm->scratch.addr == 0); - - scratch_pte = vm->pte_encode(vm->scratch.addr, - I915_CACHE_LLC, true, 0); - - pt_vaddr = kmap_atomic(pt->page); - - for (i = 0; i < GEN6_PTES; i++) - pt_vaddr[i] = scratch_pte; - - kunmap_atomic(pt_vaddr); + kunmap_px(ppgtt, pt_vaddr); } static int gen6_alloc_va_range(struct i915_address_space *vm, - uint64_t start, uint64_t length) + uint64_t start_in, uint64_t length_in) { DECLARE_BITMAP(new_page_tables, I915_PDES); struct drm_device *dev = vm->dev; @@ -1305,11 +1870,15 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); struct i915_page_table *pt; - const uint32_t start_save = start, length_save = length; + uint32_t start, length, start_save, length_save; uint32_t pde, temp; int ret; - WARN_ON(upper_32_bits(start)); + if (WARN_ON(start_in + length_in > ppgtt->base.total)) + return -ENODEV; + + start = start_save = start_in; + length = length_save = length_in; bitmap_zero(new_page_tables, I915_PDES); @@ -1319,7 +1888,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, * tables. */ gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { - if (pt != ppgtt->scratch_pt) { + if (pt != vm->scratch_pt) { WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); continue; } @@ -1327,7 +1896,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, /* We've already allocated a page table */ WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); - pt = alloc_pt_single(dev); + pt = alloc_pt(dev); if (IS_ERR(pt)) { ret = PTR_ERR(pt); goto unwind_out; @@ -1336,7 +1905,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, gen6_initialize_pt(vm, pt); ppgtt->pd.page_table[pde] = pt; - set_bit(pde, new_page_tables); + __set_bit(pde, new_page_tables); trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); } @@ -1350,7 +1919,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, bitmap_set(tmp_bitmap, gen6_pte_index(start), gen6_pte_count(start, length)); - if (test_and_clear_bit(pde, new_page_tables)) + if (__test_and_clear_bit(pde, new_page_tables)) gen6_write_pde(&ppgtt->pd, pde, pt); trace_i915_page_table_entry_map(vm, pde, pt, @@ -1374,14 +1943,41 @@ unwind_out: for_each_set_bit(pde, new_page_tables, I915_PDES) { struct i915_page_table *pt = ppgtt->pd.page_table[pde]; - ppgtt->pd.page_table[pde] = ppgtt->scratch_pt; - unmap_and_free_pt(pt, vm->dev); + ppgtt->pd.page_table[pde] = vm->scratch_pt; + free_pt(vm->dev, pt); } mark_tlbs_dirty(ppgtt); return ret; } +static int gen6_init_scratch(struct i915_address_space *vm) +{ + struct drm_device *dev = vm->dev; + + vm->scratch_page = alloc_scratch_page(dev); + if (IS_ERR(vm->scratch_page)) + return PTR_ERR(vm->scratch_page); + + vm->scratch_pt = alloc_pt(dev); + if (IS_ERR(vm->scratch_pt)) { + free_scratch_page(dev, vm->scratch_page); + return PTR_ERR(vm->scratch_pt); + } + + gen6_initialize_pt(vm, vm->scratch_pt); + + return 0; +} + +static void gen6_free_scratch(struct i915_address_space *vm) +{ + struct drm_device *dev = vm->dev; + + free_pt(dev, vm->scratch_pt); + free_scratch_page(dev, vm->scratch_page); +} + static void gen6_ppgtt_cleanup(struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt = @@ -1389,20 +1985,19 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) struct i915_page_table *pt; uint32_t pde; - drm_mm_remove_node(&ppgtt->node); gen6_for_all_pdes(pt, ppgtt, pde) { - if (pt != ppgtt->scratch_pt) - unmap_and_free_pt(pt, ppgtt->base.dev); + if (pt != vm->scratch_pt) + free_pt(ppgtt->base.dev, pt); } - unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev); - unmap_and_free_pd(&ppgtt->pd, ppgtt->base.dev); + gen6_free_scratch(vm); } static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) { + struct i915_address_space *vm = &ppgtt->base; struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; bool retried = false; @@ -1413,11 +2008,10 @@ static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) * size. We allocate at the top of the GTT to avoid fragmentation. */ BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); - ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev); - if (IS_ERR(ppgtt->scratch_pt)) - return PTR_ERR(ppgtt->scratch_pt); - gen6_initialize_pt(&ppgtt->base, ppgtt->scratch_pt); + ret = gen6_init_scratch(vm); + if (ret) + return ret; alloc: ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, @@ -1448,7 +2042,7 @@ alloc: return 0; err_out: - unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev); + gen6_free_scratch(vm); return ret; } @@ -1464,7 +2058,7 @@ static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, uint32_t pde, temp; gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) - ppgtt->pd.page_table[pde] = ppgtt->scratch_pt; + ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; } static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) @@ -1500,11 +2094,11 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; ppgtt->debug_dump = gen6_dump_ppgtt; - ppgtt->pd.pd_offset = + ppgtt->pd.base.ggtt_offset = ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset / sizeof(gen6_pte_t); + ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); @@ -1515,23 +2109,31 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->node.start / PAGE_SIZE); DRM_DEBUG("Adding PPGTT at offset %x\n", - ppgtt->pd.pd_offset << 10); + ppgtt->pd.base.ggtt_offset << 10); return 0; } static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) { - struct drm_i915_private *dev_priv = dev->dev_private; - ppgtt->base.dev = dev; - ppgtt->base.scratch = dev_priv->gtt.base.scratch; if (INTEL_INFO(dev)->gen < 8) return gen6_ppgtt_init(ppgtt); else return gen8_ppgtt_init(ppgtt); } + +static void i915_address_space_init(struct i915_address_space *vm, + struct drm_i915_private *dev_priv) +{ + drm_mm_init(&vm->mm, vm->start, vm->total); + vm->dev = dev_priv->dev; + INIT_LIST_HEAD(&vm->active_list); + INIT_LIST_HEAD(&vm->inactive_list); + list_add_tail(&vm->global_link, &dev_priv->vm_list); +} + int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -1540,9 +2142,7 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) ret = __hw_ppgtt_init(dev, ppgtt); if (ret == 0) { kref_init(&ppgtt->ref); - drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, - ppgtt->base.total); - i915_init_vm(dev_priv, &ppgtt->base); + i915_address_space_init(&ppgtt->base, dev_priv); } return ret; @@ -1550,11 +2150,6 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) int i915_ppgtt_init_hw(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - int i, ret = 0; - /* In the case of execlists, PPGTT is enabled by the context descriptor * and the PDPs are contained within the context itself. We don't * need to do anything here. */ @@ -1573,16 +2168,23 @@ int i915_ppgtt_init_hw(struct drm_device *dev) else MISSING_CASE(INTEL_INFO(dev)->gen); - if (ppgtt) { - for_each_ring(ring, dev_priv, i) { - ret = ppgtt->switch_mm(ppgtt, ring); - if (ret != 0) - return ret; - } - } + return 0; +} - return ret; +int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) +{ + struct drm_i915_private *dev_priv = req->ring->dev->dev_private; + struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; + + if (i915.enable_execlists) + return 0; + + if (!ppgtt) + return 0; + + return ppgtt->switch_mm(ppgtt, req); } + struct i915_hw_ppgtt * i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) { @@ -1843,7 +2445,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, first_entry, num_entries, max_entries)) num_entries = max_entries; - scratch_pte = gen8_pte_encode(vm->scratch.addr, + scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), I915_CACHE_LLC, use_scratch); for (i = 0; i < num_entries; i++) @@ -1869,7 +2471,8 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, first_entry, num_entries, max_entries)) num_entries = max_entries; - scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0); + scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), + I915_CACHE_LLC, use_scratch, 0); for (i = 0; i < num_entries; i++) iowrite32(scratch_pte, >t_base[i]); @@ -1902,6 +2505,56 @@ static int ggtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_gem_object *obj = vma->obj; + u32 pte_flags = 0; + int ret; + + ret = i915_get_ggtt_vma_pages(vma); + if (ret) + return ret; + + /* Currently applicable only to VLV */ + if (obj->gt_ro) + pte_flags |= PTE_READ_ONLY; + + vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, + vma->node.start, + cache_level, pte_flags); + + /* + * Without aliasing PPGTT there's no difference between + * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally + * upgrade to both bound if we bind either to avoid double-binding. + */ + vma->bound |= GLOBAL_BIND | LOCAL_BIND; + + return 0; +} + +struct ggtt_bind_vma__cb { + struct i915_vma *vma; + enum i915_cache_level cache_level; + u32 flags; +}; + +static int ggtt_bind_vma__cb(void *_arg) +{ + struct ggtt_bind_vma__cb *arg = _arg; + return ggtt_bind_vma(arg->vma, arg->cache_level, arg->flags); +} + +static int ggtt_bind_vma__BKL(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) +{ + struct ggtt_bind_vma__cb arg = { vma, cache_level, flags }; + return stop_machine(ggtt_bind_vma__cb, &arg, NULL); +} + +static int aliasing_gtt_bind_vma(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) +{ struct drm_device *dev = vma->vm->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj = vma->obj; @@ -1919,24 +2572,13 @@ static int ggtt_bind_vma(struct i915_vma *vma, pte_flags |= PTE_READ_ONLY; - if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) { + if (flags & GLOBAL_BIND) { vma->vm->insert_entries(vma->vm, pages, vma->node.start, cache_level, pte_flags); - - /* Note the inconsistency here is due to absence of the - * aliasing ppgtt on gen4 and earlier. Though we always - * request PIN_USER for execbuffer (translated to LOCAL_BIND), - * without the appgtt, we cannot honour that request and so - * must substitute it with a global binding. Since we do this - * behind the upper layers back, we need to explicitly set - * the bound flag ourselves. - */ - vma->bound |= GLOBAL_BIND; - } - if (dev_priv->mm.aliasing_ppgtt && flags & LOCAL_BIND) { + if (flags & LOCAL_BIND) { struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; appgtt->base.insert_entries(&appgtt->base, pages, vma->node.start, @@ -2004,9 +2646,9 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node, } static int i915_gem_setup_global_gtt(struct drm_device *dev, - unsigned long start, - unsigned long mappable_end, - unsigned long end) + u64 start, + u64 mappable_end, + u64 end) { /* Let GEM Manage all of the aperture. * @@ -2026,11 +2668,13 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, BUG_ON(mappable_end > end); - /* Subtract the guard page ... */ - drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE); + ggtt_vm->start = start; - dev_priv->gtt.base.start = start; - dev_priv->gtt.base.total = end - start; + /* Subtract the guard page before address space initialization to + * shrink the range used by drm_mm */ + ggtt_vm->total = end - start - PAGE_SIZE; + i915_address_space_init(ggtt_vm, dev_priv); + ggtt_vm->total += PAGE_SIZE; if (intel_vgpu_active(dev)) { ret = intel_vgt_balloon(dev); @@ -2039,13 +2683,13 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, } if (!HAS_LLC(dev)) - dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust; + ggtt_vm->mm.color_adjust = i915_gtt_color_adjust; /* Mark any preallocated objects as occupied */ list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); - DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n", + DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n", i915_gem_obj_ggtt_offset(obj), obj->base.size); WARN_ON(i915_gem_obj_ggtt_bound(obj)); @@ -2055,6 +2699,7 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, return ret; } vma->bound |= GLOBAL_BIND; + list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list); } /* Clear any non-preallocated blocks */ @@ -2097,6 +2742,8 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, true); dev_priv->mm.aliasing_ppgtt = ppgtt; + WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); + dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; } return 0; @@ -2105,7 +2752,7 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, void i915_gem_init_global_gtt(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long gtt_size, mappable_size; + u64 gtt_size, mappable_size; gtt_size = dev_priv->gtt.base.total; mappable_size = dev_priv->gtt.mappable_end; @@ -2135,42 +2782,6 @@ void i915_global_gtt_cleanup(struct drm_device *dev) vm->cleanup(vm); } -static int setup_scratch_page(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct page *page; - dma_addr_t dma_addr; - - page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); - if (page == NULL) - return -ENOMEM; - set_pages_uc(page, 1); - -#ifdef CONFIG_INTEL_IOMMU - dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(dev->pdev, dma_addr)) - return -EINVAL; -#else - dma_addr = page_to_phys(page); -#endif - dev_priv->gtt.base.scratch.page = page; - dev_priv->gtt.base.scratch.addr = dma_addr; - - return 0; -} - -static void teardown_scratch_page(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct page *page = dev_priv->gtt.base.scratch.page; - - set_pages_wb(page, 1); - pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - __free_page(page); -} - static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) { snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; @@ -2253,8 +2864,8 @@ static int ggtt_probe_common(struct drm_device *dev, size_t gtt_size) { struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_page_scratch *scratch_page; phys_addr_t gtt_phys_addr; - int ret; /* For Modern GENs the PTEs and register space are split in the BAR */ gtt_phys_addr = pci_resource_start(dev->pdev, 0) + @@ -2276,14 +2887,17 @@ static int ggtt_probe_common(struct drm_device *dev, return -ENOMEM; } - ret = setup_scratch_page(dev); - if (ret) { + scratch_page = alloc_scratch_page(dev); + if (IS_ERR(scratch_page)) { DRM_ERROR("Scratch setup failed\n"); /* iounmap will also get called at remove, but meh */ iounmap(dev_priv->gtt.gsm); + return PTR_ERR(scratch_page); } - return ret; + dev_priv->gtt.base.scratch_page = scratch_page; + + return 0; } /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability @@ -2320,8 +2934,8 @@ static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b * write would work. */ - I915_WRITE(GEN8_PRIVATE_PAT, pat); - I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); + I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); + I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); } static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) @@ -2355,18 +2969,18 @@ static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) GEN8_PPAT(6, CHV_PPAT_SNOOP) | GEN8_PPAT(7, CHV_PPAT_SNOOP); - I915_WRITE(GEN8_PRIVATE_PAT, pat); - I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); + I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); + I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); } static int gen8_gmch_probe(struct drm_device *dev, - size_t *gtt_total, + u64 *gtt_total, size_t *stolen, phys_addr_t *mappable_base, - unsigned long *mappable_end) + u64 *mappable_end) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned int gtt_size; + u64 gtt_size; u16 snb_gmch_ctl; int ret; @@ -2404,14 +3018,17 @@ static int gen8_gmch_probe(struct drm_device *dev, dev_priv->gtt.base.bind_vma = ggtt_bind_vma; dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; + if (IS_CHERRYVIEW(dev)) + dev_priv->gtt.base.bind_vma = ggtt_bind_vma__BKL; + return ret; } static int gen6_gmch_probe(struct drm_device *dev, - size_t *gtt_total, + u64 *gtt_total, size_t *stolen, phys_addr_t *mappable_base, - unsigned long *mappable_end) + u64 *mappable_end) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned int gtt_size; @@ -2425,7 +3042,7 @@ static int gen6_gmch_probe(struct drm_device *dev, * a coarse sanity check. */ if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { - DRM_ERROR("Unknown GMADR size (%lx)\n", + DRM_ERROR("Unknown GMADR size (%llx)\n", dev_priv->gtt.mappable_end); return -ENXIO; } @@ -2455,14 +3072,14 @@ static void gen6_gmch_remove(struct i915_address_space *vm) struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); iounmap(gtt->gsm); - teardown_scratch_page(vm->dev); + free_scratch_page(vm->dev, vm->scratch_page); } static int i915_gmch_probe(struct drm_device *dev, - size_t *gtt_total, + u64 *gtt_total, size_t *stolen, phys_addr_t *mappable_base, - unsigned long *mappable_end) + u64 *mappable_end) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; @@ -2519,17 +3136,17 @@ int i915_gem_gtt_init(struct drm_device *dev) dev_priv->gtt.base.cleanup = gen6_gmch_remove; } + gtt->base.dev = dev; + ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, >t->mappable_base, >t->mappable_end); if (ret) return ret; - gtt->base.dev = dev; - /* GMADR is the PCI mmio aperture into the global GTT. */ - DRM_INFO("Memory usable by graphics device = %zdM\n", + DRM_INFO("Memory usable by graphics device = %lluM\n", gtt->base.total >> 20); - DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20); + DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20); DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); #ifdef CONFIG_INTEL_IOMMU if (intel_iommu_gfx_mapped) @@ -2675,15 +3292,18 @@ i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, } -static void -rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height, - struct sg_table *st) +static struct scatterlist * +rotate_pages(dma_addr_t *in, unsigned int offset, + unsigned int width, unsigned int height, + struct sg_table *st, struct scatterlist *sg) { unsigned int column, row; unsigned int src_idx; - struct scatterlist *sg = st->sgl; - st->nents = 0; + if (!sg) { + st->nents = 0; + sg = st->sgl; + } for (column = 0; column < width; column++) { src_idx = width * (height - 1) + column; @@ -2694,51 +3314,49 @@ rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height, * The only thing we need are DMA addresses. */ sg_set_page(sg, NULL, PAGE_SIZE, 0); - sg_dma_address(sg) = in[src_idx]; + sg_dma_address(sg) = in[offset + src_idx]; sg_dma_len(sg) = PAGE_SIZE; sg = sg_next(sg); src_idx -= width; } } + + return sg; } static struct sg_table * intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->base.dev; - struct intel_rotation_info *rot_info = &ggtt_view->rotation_info; - unsigned long size, pages, rot_pages; + struct intel_rotation_info *rot_info = &ggtt_view->params.rotation_info; + unsigned int size_pages = rot_info->size >> PAGE_SHIFT; + unsigned int size_pages_uv; struct sg_page_iter sg_iter; unsigned long i; dma_addr_t *page_addr_list; struct sg_table *st; - unsigned int tile_pitch, tile_height; - unsigned int width_pages, height_pages; + unsigned int uv_start_page; + struct scatterlist *sg; int ret = -ENOMEM; - pages = obj->base.size / PAGE_SIZE; - - /* Calculate tiling geometry. */ - tile_height = intel_tile_height(dev, rot_info->pixel_format, - rot_info->fb_modifier); - tile_pitch = PAGE_SIZE / tile_height; - width_pages = DIV_ROUND_UP(rot_info->pitch, tile_pitch); - height_pages = DIV_ROUND_UP(rot_info->height, tile_height); - rot_pages = width_pages * height_pages; - size = rot_pages * PAGE_SIZE; - /* Allocate a temporary list of source pages for random access. */ - page_addr_list = drm_malloc_ab(pages, sizeof(dma_addr_t)); + page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, + sizeof(dma_addr_t)); if (!page_addr_list) return ERR_PTR(ret); + /* Account for UV plane with NV12. */ + if (rot_info->pixel_format == DRM_FORMAT_NV12) + size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; + else + size_pages_uv = 0; + /* Allocate target SG list. */ st = kmalloc(sizeof(*st), GFP_KERNEL); if (!st) goto err_st_alloc; - ret = sg_alloc_table(st, rot_pages, GFP_KERNEL); + ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL); if (ret) goto err_sg_alloc; @@ -2750,13 +3368,32 @@ intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, } /* Rotate the pages. */ - rotate_pages(page_addr_list, width_pages, height_pages, st); + sg = rotate_pages(page_addr_list, 0, + rot_info->width_pages, rot_info->height_pages, + st, NULL); + + /* Append the UV plane if NV12. */ + if (rot_info->pixel_format == DRM_FORMAT_NV12) { + uv_start_page = size_pages; + + /* Check for tile-row un-alignment. */ + if (offset_in_page(rot_info->uv_offset)) + uv_start_page--; + + rot_info->uv_start_page = uv_start_page; + + rotate_pages(page_addr_list, uv_start_page, + rot_info->width_pages_uv, + rot_info->height_pages_uv, + st, sg); + } DRM_DEBUG_KMS( - "Created rotated page mapping for object size %lu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %lu pages).\n", - size, rot_info->pitch, rot_info->height, - rot_info->pixel_format, width_pages, height_pages, - rot_pages); + "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", + obj->base.size, rot_info->pitch, rot_info->height, + rot_info->pixel_format, rot_info->width_pages, + rot_info->height_pages, size_pages + size_pages_uv, + size_pages); drm_free_large(page_addr_list); @@ -2768,10 +3405,11 @@ err_st_alloc: drm_free_large(page_addr_list); DRM_DEBUG_KMS( - "Failed to create rotated mapping for object size %lu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %lu pages)\n", - size, ret, rot_info->pitch, rot_info->height, - rot_info->pixel_format, width_pages, height_pages, - rot_pages); + "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", + obj->base.size, ret, rot_info->pitch, rot_info->height, + rot_info->pixel_format, rot_info->width_pages, + rot_info->height_pages, size_pages + size_pages_uv, + size_pages); return ERR_PTR(ret); } @@ -2889,9 +3527,12 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, vma->node.size, VM_TO_TRACE_NAME(vma->vm)); + /* XXX: i915_vma_pin() will fix this +- hack */ + vma->pin_count++; ret = vma->vm->allocate_va_range(vma->vm, vma->node.start, vma->node.size); + vma->pin_count--; if (ret) return ret; } @@ -2916,9 +3557,10 @@ size_t i915_ggtt_view_size(struct drm_i915_gem_object *obj, const struct i915_ggtt_view *view) { - if (view->type == I915_GGTT_VIEW_NORMAL || - view->type == I915_GGTT_VIEW_ROTATED) { + if (view->type == I915_GGTT_VIEW_NORMAL) { return obj->base.size; + } else if (view->type == I915_GGTT_VIEW_ROTATED) { + return view->params.rotation_info.size; } else if (view->type == I915_GGTT_VIEW_PARTIAL) { return view->params.partial.size << PAGE_SHIFT; } else { diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 0d46dd20bf71..877c32c78a6a 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -39,6 +39,8 @@ struct drm_i915_file_private; typedef uint32_t gen6_pte_t; typedef uint64_t gen8_pte_t; typedef uint64_t gen8_pde_t; +typedef uint64_t gen8_ppgtt_pdpe_t; +typedef uint64_t gen8_ppgtt_pml4e_t; #define gtt_total_entries(gtt) ((gtt).base.total >> PAGE_SHIFT) @@ -88,9 +90,18 @@ typedef uint64_t gen8_pde_t; * PDPE | PDE | PTE | offset * The difference as compared to normal x86 3 level page table is the PDPEs are * programmed via register. + * + * GEN8 48b legacy style address is defined as a 4 level page table: + * 47:39 | 38:30 | 29:21 | 20:12 | 11:0 + * PML4E | PDPE | PDE | PTE | offset */ +#define GEN8_PML4ES_PER_PML4 512 +#define GEN8_PML4E_SHIFT 39 +#define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1) #define GEN8_PDPE_SHIFT 30 -#define GEN8_PDPE_MASK 0x3 +/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page + * tables */ +#define GEN8_PDPE_MASK 0x1ff #define GEN8_PDE_SHIFT 21 #define GEN8_PDE_MASK 0x1ff #define GEN8_PTE_SHIFT 12 @@ -98,6 +109,9 @@ typedef uint64_t gen8_pde_t; #define GEN8_LEGACY_PDPES 4 #define GEN8_PTES I915_PTES(sizeof(gen8_pte_t)) +#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ + GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES) + #define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD) #define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */ #define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */ @@ -124,8 +138,14 @@ enum i915_ggtt_view_type { struct intel_rotation_info { unsigned int height; unsigned int pitch; + unsigned int uv_offset; uint32_t pixel_format; uint64_t fb_modifier; + unsigned int width_pages, height_pages; + uint64_t size; + unsigned int width_pages_uv, height_pages_uv; + uint64_t size_uv; + unsigned int uv_start_page; }; struct i915_ggtt_view { @@ -133,16 +153,13 @@ struct i915_ggtt_view { union { struct { - unsigned long offset; + u64 offset; unsigned int size; } partial; + struct intel_rotation_info rotation_info; } params; struct sg_table *pages; - - union { - struct intel_rotation_info rotation_info; - }; }; extern const struct i915_ggtt_view i915_ggtt_view_normal; @@ -205,41 +222,64 @@ struct i915_vma { #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf }; -struct i915_page_table { +struct i915_page_dma { struct page *page; - dma_addr_t daddr; + union { + dma_addr_t daddr; + + /* For gen6/gen7 only. This is the offset in the GGTT + * where the page directory entries for PPGTT begin + */ + uint32_t ggtt_offset; + }; +}; + +#define px_base(px) (&(px)->base) +#define px_page(px) (px_base(px)->page) +#define px_dma(px) (px_base(px)->daddr) + +struct i915_page_scratch { + struct i915_page_dma base; +}; + +struct i915_page_table { + struct i915_page_dma base; unsigned long *used_ptes; }; struct i915_page_directory { - struct page *page; /* NULL for GEN6-GEN7 */ - union { - uint32_t pd_offset; - dma_addr_t daddr; - }; + struct i915_page_dma base; unsigned long *used_pdes; struct i915_page_table *page_table[I915_PDES]; /* PDEs */ }; struct i915_page_directory_pointer { - /* struct page *page; */ - DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES); - struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES]; + struct i915_page_dma base; + + unsigned long *used_pdpes; + struct i915_page_directory **page_directory; +}; + +struct i915_pml4 { + struct i915_page_dma base; + + DECLARE_BITMAP(used_pml4es, GEN8_PML4ES_PER_PML4); + struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4]; }; struct i915_address_space { struct drm_mm mm; struct drm_device *dev; struct list_head global_link; - unsigned long start; /* Start offset always 0 for dri2 */ - size_t total; /* size addr space maps (ex. 2GB for ggtt) */ + u64 start; /* Start offset always 0 for dri2 */ + u64 total; /* size addr space maps (ex. 2GB for ggtt) */ - struct { - dma_addr_t addr; - struct page *page; - } scratch; + struct i915_page_scratch *scratch_page; + struct i915_page_table *scratch_pt; + struct i915_page_directory *scratch_pd; + struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */ /** * List of objects currently involved in rendering. @@ -300,9 +340,10 @@ struct i915_address_space { */ struct i915_gtt { struct i915_address_space base; - size_t stolen_size; /* Total size of stolen memory */ - unsigned long mappable_end; /* End offset that we can CPU map */ + size_t stolen_size; /* Total size of stolen memory */ + size_t stolen_usable_size; /* Total size minus BIOS reserved */ + u64 mappable_end; /* End offset that we can CPU map */ struct io_mapping *mappable; /* Mapping to our CPU mappable region */ phys_addr_t mappable_base; /* PA of our GMADR */ @@ -314,9 +355,9 @@ struct i915_gtt { int mtrr; /* global gtt ops */ - int (*gtt_probe)(struct drm_device *dev, size_t *gtt_total, + int (*gtt_probe)(struct drm_device *dev, u64 *gtt_total, size_t *stolen, phys_addr_t *mappable_base, - unsigned long *mappable_end); + u64 *mappable_end); }; struct i915_hw_ppgtt { @@ -325,20 +366,18 @@ struct i915_hw_ppgtt { struct drm_mm_node node; unsigned long pd_dirty_rings; union { - struct i915_page_directory_pointer pdp; - struct i915_page_directory pd; + struct i915_pml4 pml4; /* GEN8+ & 48b PPGTT */ + struct i915_page_directory_pointer pdp; /* GEN8+ */ + struct i915_page_directory pd; /* GEN6-7 */ }; - struct i915_page_table *scratch_pt; - struct i915_page_directory *scratch_pd; - struct drm_i915_file_private *file_priv; gen6_pte_t __iomem *pd_addr; int (*enable)(struct i915_hw_ppgtt *ppgtt); int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, - struct intel_engine_cs *ring); + struct drm_i915_gem_request *req); void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); }; @@ -352,7 +391,8 @@ struct i915_hw_ppgtt { */ #define gen6_for_each_pde(pt, pd, start, length, temp, iter) \ for (iter = gen6_pde_index(start); \ - pt = (pd)->page_table[iter], length > 0 && iter < I915_PDES; \ + length > 0 && iter < I915_PDES ? \ + (pt = (pd)->page_table[iter]), 1 : 0; \ iter++, \ temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT) - start, \ temp = min_t(unsigned, temp, length), \ @@ -417,30 +457,30 @@ static inline uint32_t gen6_pde_index(uint32_t addr) */ #define gen8_for_each_pde(pt, pd, start, length, temp, iter) \ for (iter = gen8_pde_index(start); \ - pt = (pd)->page_table[iter], length > 0 && iter < I915_PDES; \ + length > 0 && iter < I915_PDES ? \ + (pt = (pd)->page_table[iter]), 1 : 0; \ iter++, \ temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT) - start, \ temp = min(temp, length), \ start += temp, length -= temp) -#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \ - for (iter = gen8_pdpe_index(start); \ - pd = (pdp)->page_directory[iter], length > 0 && iter < GEN8_LEGACY_PDPES; \ +#define gen8_for_each_pdpe(pd, pdp, start, length, temp, iter) \ + for (iter = gen8_pdpe_index(start); \ + length > 0 && (iter < I915_PDPES_PER_PDP(dev)) ? \ + (pd = (pdp)->page_directory[iter]), 1 : 0; \ iter++, \ temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start, \ temp = min(temp, length), \ start += temp, length -= temp) -/* Clamp length to the next page_directory boundary */ -static inline uint64_t gen8_clamp_pd(uint64_t start, uint64_t length) -{ - uint64_t next_pd = ALIGN(start + 1, 1 << GEN8_PDPE_SHIFT); - - if (next_pd > (start + length)) - return length; - - return next_pd - start; -} +#define gen8_for_each_pml4e(pdp, pml4, start, length, temp, iter) \ + for (iter = gen8_pml4e_index(start); \ + length > 0 && iter < GEN8_PML4ES_PER_PML4 ? \ + (pdp = (pml4)->pdps[iter]), 1 : 0; \ + iter++, \ + temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT) - start, \ + temp = min(temp, length), \ + start += temp, length -= temp) static inline uint32_t gen8_pte_index(uint64_t address) { @@ -459,8 +499,7 @@ static inline uint32_t gen8_pdpe_index(uint64_t address) static inline uint32_t gen8_pml4e_index(uint64_t address) { - WARN_ON(1); /* For 64B */ - return 0; + return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK; } static inline size_t gen8_pte_count(uint64_t address, uint64_t length) @@ -468,6 +507,14 @@ static inline size_t gen8_pte_count(uint64_t address, uint64_t length) return i915_pte_count(address, length, GEN8_PDE_SHIFT); } +static inline dma_addr_t +i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n) +{ + return test_bit(n, ppgtt->pdp.used_pdpes) ? + px_dma(ppgtt->pdp.page_directory[n]) : + px_dma(ppgtt->base.scratch_pd); +} + int i915_gem_gtt_init(struct drm_device *dev); void i915_gem_init_global_gtt(struct drm_device *dev); void i915_global_gtt_cleanup(struct drm_device *dev); @@ -475,6 +522,7 @@ void i915_global_gtt_cleanup(struct drm_device *dev); int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt); int i915_ppgtt_init_hw(struct drm_device *dev); +int i915_ppgtt_init_ring(struct drm_i915_gem_request *req); void i915_ppgtt_release(struct kref *kref); struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv); @@ -505,7 +553,7 @@ i915_ggtt_view_equal(const struct i915_ggtt_view *a, if (a->type != b->type) return false; - if (a->type == I915_GGTT_VIEW_PARTIAL) + if (a->type != I915_GGTT_VIEW_NORMAL) return !memcmp(&a->params, &b->params, sizeof(a->params)); return true; } diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c index 521548a08578..5026a6267a88 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.c +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -73,6 +73,24 @@ free_gem: return ret; } +/* + * Macro to add commands to auxiliary batch. + * This macro only checks for page overflow before inserting the commands, + * this is sufficient as the null state generator makes the final batch + * with two passes to build command and state separately. At this point + * the size of both are known and it compacts them by relocating the state + * right after the commands taking care of aligment so we should sufficient + * space below them for adding new commands. + */ +#define OUT_BATCH(batch, i, val) \ + do { \ + if (WARN_ON((i) >= PAGE_SIZE / sizeof(u32))) { \ + ret = -ENOSPC; \ + goto err_out; \ + } \ + (batch)[(i)++] = (val); \ + } while(0) + static int render_state_setup(struct render_state *so) { const struct intel_renderstate_rodata *rodata = so->rodata; @@ -96,8 +114,10 @@ static int render_state_setup(struct render_state *so) s = lower_32_bits(r); if (so->gen >= 8) { if (i + 1 >= rodata->batch_items || - rodata->batch[i + 1] != 0) - return -EINVAL; + rodata->batch[i + 1] != 0) { + ret = -EINVAL; + goto err_out; + } d[i++] = s; s = upper_32_bits(r); @@ -108,6 +128,21 @@ static int render_state_setup(struct render_state *so) d[i++] = s; } + + while (i % CACHELINE_DWORDS) + OUT_BATCH(d, i, MI_NOOP); + + so->aux_batch_offset = i * sizeof(u32); + + OUT_BATCH(d, i, MI_BATCH_BUFFER_END); + so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset; + + /* + * Since we are sending length, we need to strictly conform to + * all requirements. For Gen2 this must be a multiple of 8. + */ + so->aux_batch_size = ALIGN(so->aux_batch_size, 8); + kunmap(page); ret = i915_gem_object_set_to_gtt_domain(so->obj, false); @@ -120,8 +155,14 @@ static int render_state_setup(struct render_state *so) } return 0; + +err_out: + kunmap(page); + return ret; } +#undef OUT_BATCH + void i915_gem_render_state_fini(struct render_state *so) { i915_gem_object_ggtt_unpin(so->obj); @@ -152,29 +193,36 @@ int i915_gem_render_state_prepare(struct intel_engine_cs *ring, return 0; } -int i915_gem_render_state_init(struct intel_engine_cs *ring) +int i915_gem_render_state_init(struct drm_i915_gem_request *req) { struct render_state so; int ret; - ret = i915_gem_render_state_prepare(ring, &so); + ret = i915_gem_render_state_prepare(req->ring, &so); if (ret) return ret; if (so.rodata == NULL) return 0; - ret = ring->dispatch_execbuffer(ring, - so.ggtt_offset, - so.rodata->batch_items * 4, - I915_DISPATCH_SECURE); + ret = req->ring->dispatch_execbuffer(req, so.ggtt_offset, + so.rodata->batch_items * 4, + I915_DISPATCH_SECURE); if (ret) goto out; - i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring); + if (so.aux_batch_size > 8) { + ret = req->ring->dispatch_execbuffer(req, + (so.ggtt_offset + + so.aux_batch_offset), + so.aux_batch_size, + I915_DISPATCH_SECURE); + if (ret) + goto out; + } + + i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req); - ret = __i915_add_request(ring, NULL, so.obj); - /* __i915_add_request moves object to inactive if it fails */ out: i915_gem_render_state_fini(&so); return ret; diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h index c44961ed3fad..e641bb093a90 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.h +++ b/drivers/gpu/drm/i915/i915_gem_render_state.h @@ -37,9 +37,11 @@ struct render_state { struct drm_i915_gem_object *obj; u64 ggtt_offset; int gen; + u32 aux_batch_size; + u32 aux_batch_offset; }; -int i915_gem_render_state_init(struct intel_engine_cs *ring); +int i915_gem_render_state_init(struct drm_i915_gem_request *req); void i915_gem_render_state_fini(struct render_state *so); int i915_gem_render_state_prepare(struct intel_engine_cs *ring, struct render_state *so); diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index f6ecbda2c604..f7df54a8ee2b 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -73,7 +73,7 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, - long target, unsigned flags) + unsigned long target, unsigned flags) { const struct { struct list_head *list; @@ -85,6 +85,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, }, *phase; unsigned long count = 0; + trace_i915_gem_shrink(dev_priv, target, flags); + i915_gem_retire_requests(dev_priv->dev); + /* * As we may completely rewrite the (un)bound list whilst unbinding * (due to retiring requests) we have to strictly process only @@ -123,6 +126,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, obj->madv != I915_MADV_DONTNEED) continue; + if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active) + continue; + drm_gem_object_reference(&obj->base); /* For the unbound phase, this should be a no-op! */ @@ -139,11 +145,13 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, list_splice(&still_in_list, phase->list); } + i915_gem_retire_requests(dev_priv->dev); + return count; } /** - * i915_gem_shrink - Shrink buffer object caches completely + * i915_gem_shrink_all - Shrink buffer object caches completely * @dev_priv: i915 device * * This is a simple wraper around i915_gem_shrink() to aggressively shrink all @@ -158,9 +166,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, */ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) { - i915_gem_evict_everything(dev_priv->dev); - return i915_gem_shrink(dev_priv, LONG_MAX, - I915_SHRINK_BOUND | I915_SHRINK_UNBOUND); + return i915_gem_shrink(dev_priv, -1UL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_ACTIVE); } static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) @@ -213,7 +222,7 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) count += obj->base.size >> PAGE_SHIFT; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { - if (obj->pages_pin_count == num_vma_bound(obj)) + if (!obj->active && obj->pages_pin_count == num_vma_bound(obj)) count += obj->base.size >> PAGE_SHIFT; } diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 8b5b784c62fe..598ed2facf85 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -30,6 +30,9 @@ #include <drm/i915_drm.h> #include "i915_drv.h" +#define KB(x) ((x) * 1024) +#define MB(x) (KB(x) * 1024) + /* * The BIOS typically reserves some of the system's memory for the exclusive * use of the integrated graphics. This memory is no longer available for @@ -42,6 +45,46 @@ * for is a boon. */ +int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, + struct drm_mm_node *node, u64 size, + unsigned alignment, u64 start, u64 end) +{ + int ret; + + if (!drm_mm_initialized(&dev_priv->mm.stolen)) + return -ENODEV; + + /* See the comment at the drm_mm_init() call for more about this check. + * WaSkipStolenMemoryFirstPage:bdw,chv (incomplete) */ + if (INTEL_INFO(dev_priv)->gen == 8 && start < 4096) + start = 4096; + + mutex_lock(&dev_priv->mm.stolen_lock); + ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node, size, + alignment, start, end, + DRM_MM_SEARCH_DEFAULT); + mutex_unlock(&dev_priv->mm.stolen_lock); + + return ret; +} + +int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv, + struct drm_mm_node *node, u64 size, + unsigned alignment) +{ + return i915_gem_stolen_insert_node_in_range(dev_priv, node, size, + alignment, 0, + dev_priv->gtt.stolen_usable_size); +} + +void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, + struct drm_mm_node *node) +{ + mutex_lock(&dev_priv->mm.stolen_lock); + drm_mm_remove_node(node); + mutex_unlock(&dev_priv->mm.stolen_lock); +} + static unsigned long i915_stolen_to_physical(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -51,24 +94,91 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) /* Almost universally we can find the Graphics Base of Stolen Memory * at offset 0x5c in the igfx configuration space. On a few (desktop) * machines this is also mirrored in the bridge device at different - * locations, or in the MCHBAR. On gen2, the layout is again slightly - * different with the Graphics Segment immediately following Top of - * Memory (or Top of Usable DRAM). Note it appears that TOUD is only - * reported by 865g, so we just use the top of memory as determined - * by the e820 probe. + * locations, or in the MCHBAR. + * + * On 865 we just check the TOUD register. + * + * On 830/845/85x the stolen memory base isn't available in any + * register. We need to calculate it as TOM-TSEG_SIZE-stolen_size. * - * XXX However gen2 requires an unavailable symbol. */ base = 0; if (INTEL_INFO(dev)->gen >= 3) { /* Read Graphics Base of Stolen Memory directly */ pci_read_config_dword(dev->pdev, 0x5c, &base); base &= ~((1<<20) - 1); - } else { /* GEN2 */ -#if 0 - /* Stolen is immediately above Top of Memory */ - base = max_low_pfn_mapped << PAGE_SHIFT; -#endif + } else if (IS_I865G(dev)) { + u16 toud = 0; + + /* + * FIXME is the graphics stolen memory region + * always at TOUD? Ie. is it always the last + * one to be allocated by the BIOS? + */ + pci_bus_read_config_word(dev->pdev->bus, PCI_DEVFN(0, 0), + I865_TOUD, &toud); + + base = toud << 16; + } else if (IS_I85X(dev)) { + u32 tseg_size = 0; + u32 tom; + u8 tmp; + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0), + I85X_ESMRAMC, &tmp); + + if (tmp & TSEG_ENABLE) + tseg_size = MB(1); + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 1), + I85X_DRB3, &tmp); + tom = tmp * MB(32); + + base = tom - tseg_size - dev_priv->gtt.stolen_size; + } else if (IS_845G(dev)) { + u32 tseg_size = 0; + u32 tom; + u8 tmp; + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0), + I845_ESMRAMC, &tmp); + + if (tmp & TSEG_ENABLE) { + switch (tmp & I845_TSEG_SIZE_MASK) { + case I845_TSEG_SIZE_512K: + tseg_size = KB(512); + break; + case I845_TSEG_SIZE_1M: + tseg_size = MB(1); + break; + } + } + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0), + I830_DRB3, &tmp); + tom = tmp * MB(32); + + base = tom - tseg_size - dev_priv->gtt.stolen_size; + } else if (IS_I830(dev)) { + u32 tseg_size = 0; + u32 tom; + u8 tmp; + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0), + I830_ESMRAMC, &tmp); + + if (tmp & TSEG_ENABLE) { + if (tmp & I830_TSEG_SIZE_1M) + tseg_size = MB(1); + else + tseg_size = KB(512); + } + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0), + I830_DRB3, &tmp); + tom = tmp * MB(32); + + base = tom - tseg_size - dev_priv->gtt.stolen_size; } if (base == 0) @@ -151,150 +261,138 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) return base; } -static int find_compression_threshold(struct drm_device *dev, - struct drm_mm_node *node, - int size, - int fb_cpp) +void i915_gem_cleanup_stolen(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - int compression_threshold = 1; - int ret; - - /* HACK: This code depends on what we will do in *_enable_fbc. If that - * code changes, this code needs to change as well. - * - * The enable_fbc code will attempt to use one of our 2 compression - * thresholds, therefore, in that case, we only have 1 resort. - */ - /* Try to over-allocate to reduce reallocations and fragmentation. */ - ret = drm_mm_insert_node(&dev_priv->mm.stolen, node, - size <<= 1, 4096, DRM_MM_SEARCH_DEFAULT); - if (ret == 0) - return compression_threshold; - -again: - /* HW's ability to limit the CFB is 1:4 */ - if (compression_threshold > 4 || - (fb_cpp == 2 && compression_threshold == 2)) - return 0; + if (!drm_mm_initialized(&dev_priv->mm.stolen)) + return; - ret = drm_mm_insert_node(&dev_priv->mm.stolen, node, - size >>= 1, 4096, - DRM_MM_SEARCH_DEFAULT); - if (ret && INTEL_INFO(dev)->gen <= 4) { - return 0; - } else if (ret) { - compression_threshold <<= 1; - goto again; - } else { - return compression_threshold; - } + drm_mm_takedown(&dev_priv->mm.stolen); } -static int i915_setup_compression(struct drm_device *dev, int size, int fb_cpp) +static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv, + unsigned long *base, unsigned long *size) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_mm_node *uninitialized_var(compressed_llb); - int ret; - - ret = find_compression_threshold(dev, &dev_priv->fbc.compressed_fb, - size, fb_cpp); - if (!ret) - goto err_llb; - else if (ret > 1) { - DRM_INFO("Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n"); - - } - - dev_priv->fbc.threshold = ret; - - if (INTEL_INFO(dev_priv)->gen >= 5) - I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->fbc.compressed_fb.start); - else if (IS_GM45(dev)) { - I915_WRITE(DPFC_CB_BASE, dev_priv->fbc.compressed_fb.start); - } else { - compressed_llb = kzalloc(sizeof(*compressed_llb), GFP_KERNEL); - if (!compressed_llb) - goto err_fb; - - ret = drm_mm_insert_node(&dev_priv->mm.stolen, compressed_llb, - 4096, 4096, DRM_MM_SEARCH_DEFAULT); - if (ret) - goto err_fb; - - dev_priv->fbc.compressed_llb = compressed_llb; + uint32_t reg_val = I915_READ(IS_GM45(dev_priv) ? + CTG_STOLEN_RESERVED : + ELK_STOLEN_RESERVED); + unsigned long stolen_top = dev_priv->mm.stolen_base + + dev_priv->gtt.stolen_size; + + *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16; + + WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base); + + /* On these platforms, the register doesn't have a size field, so the + * size is the distance between the base and the top of the stolen + * memory. We also have the genuine case where base is zero and there's + * nothing reserved. */ + if (*base == 0) + *size = 0; + else + *size = stolen_top - *base; +} - I915_WRITE(FBC_CFB_BASE, - dev_priv->mm.stolen_base + dev_priv->fbc.compressed_fb.start); - I915_WRITE(FBC_LL_BASE, - dev_priv->mm.stolen_base + compressed_llb->start); +static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv, + unsigned long *base, unsigned long *size) +{ + uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED); + + *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK; + + switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) { + case GEN6_STOLEN_RESERVED_1M: + *size = 1024 * 1024; + break; + case GEN6_STOLEN_RESERVED_512K: + *size = 512 * 1024; + break; + case GEN6_STOLEN_RESERVED_256K: + *size = 256 * 1024; + break; + case GEN6_STOLEN_RESERVED_128K: + *size = 128 * 1024; + break; + default: + *size = 1024 * 1024; + MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK); } - - dev_priv->fbc.uncompressed_size = size; - - DRM_DEBUG_KMS("reserved %d bytes of contiguous stolen space for FBC\n", - size); - - return 0; - -err_fb: - kfree(compressed_llb); - drm_mm_remove_node(&dev_priv->fbc.compressed_fb); -err_llb: - pr_info_once("drm: not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size); - return -ENOSPC; } -int i915_gem_stolen_setup_compression(struct drm_device *dev, int size, int fb_cpp) +static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv, + unsigned long *base, unsigned long *size) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!drm_mm_initialized(&dev_priv->mm.stolen)) - return -ENODEV; - - if (size <= dev_priv->fbc.uncompressed_size) - return 0; - - /* Release any current block */ - i915_gem_stolen_cleanup_compression(dev); - - return i915_setup_compression(dev, size, fb_cpp); + uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED); + + *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK; + + switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) { + case GEN7_STOLEN_RESERVED_1M: + *size = 1024 * 1024; + break; + case GEN7_STOLEN_RESERVED_256K: + *size = 256 * 1024; + break; + default: + *size = 1024 * 1024; + MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK); + } } -void i915_gem_stolen_cleanup_compression(struct drm_device *dev) +static void gen8_get_stolen_reserved(struct drm_i915_private *dev_priv, + unsigned long *base, unsigned long *size) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (dev_priv->fbc.uncompressed_size == 0) - return; - - drm_mm_remove_node(&dev_priv->fbc.compressed_fb); - - if (dev_priv->fbc.compressed_llb) { - drm_mm_remove_node(dev_priv->fbc.compressed_llb); - kfree(dev_priv->fbc.compressed_llb); + uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED); + + *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK; + + switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) { + case GEN8_STOLEN_RESERVED_1M: + *size = 1024 * 1024; + break; + case GEN8_STOLEN_RESERVED_2M: + *size = 2 * 1024 * 1024; + break; + case GEN8_STOLEN_RESERVED_4M: + *size = 4 * 1024 * 1024; + break; + case GEN8_STOLEN_RESERVED_8M: + *size = 8 * 1024 * 1024; + break; + default: + *size = 8 * 1024 * 1024; + MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK); } - - dev_priv->fbc.uncompressed_size = 0; } -void i915_gem_cleanup_stolen(struct drm_device *dev) +static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv, + unsigned long *base, unsigned long *size) { - struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED); + unsigned long stolen_top; - if (!drm_mm_initialized(&dev_priv->mm.stolen)) - return; + stolen_top = dev_priv->mm.stolen_base + dev_priv->gtt.stolen_size; - i915_gem_stolen_cleanup_compression(dev); - drm_mm_takedown(&dev_priv->mm.stolen); + *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK; + + /* On these platforms, the register doesn't have a size field, so the + * size is the distance between the base and the top of the stolen + * memory. We also have the genuine case where base is zero and there's + * nothing reserved. */ + if (*base == 0) + *size = 0; + else + *size = stolen_top - *base; } int i915_gem_init_stolen(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 tmp; - int bios_reserved = 0; + unsigned long reserved_total, reserved_base = 0, reserved_size; + unsigned long stolen_top; + + mutex_init(&dev_priv->mm.stolen_lock); #ifdef CONFIG_INTEL_IOMMU if (intel_iommu_gfx_mapped && INTEL_INFO(dev)->gen < 8) { @@ -310,26 +408,79 @@ int i915_gem_init_stolen(struct drm_device *dev) if (dev_priv->mm.stolen_base == 0) return 0; - DRM_DEBUG_KMS("found %zd bytes of stolen memory at %08lx\n", - dev_priv->gtt.stolen_size, dev_priv->mm.stolen_base); - - if (INTEL_INFO(dev)->gen >= 8) { - tmp = I915_READ(GEN7_BIOS_RESERVED); - tmp >>= GEN8_BIOS_RESERVED_SHIFT; - tmp &= GEN8_BIOS_RESERVED_MASK; - bios_reserved = (1024*1024) << tmp; - } else if (IS_GEN7(dev)) { - tmp = I915_READ(GEN7_BIOS_RESERVED); - bios_reserved = tmp & GEN7_BIOS_RESERVED_256K ? - 256*1024 : 1024*1024; + stolen_top = dev_priv->mm.stolen_base + dev_priv->gtt.stolen_size; + + switch (INTEL_INFO(dev_priv)->gen) { + case 2: + case 3: + break; + case 4: + if (IS_G4X(dev)) + g4x_get_stolen_reserved(dev_priv, &reserved_base, + &reserved_size); + break; + case 5: + /* Assume the gen6 maximum for the older platforms. */ + reserved_size = 1024 * 1024; + reserved_base = stolen_top - reserved_size; + break; + case 6: + gen6_get_stolen_reserved(dev_priv, &reserved_base, + &reserved_size); + break; + case 7: + gen7_get_stolen_reserved(dev_priv, &reserved_base, + &reserved_size); + break; + default: + if (IS_BROADWELL(dev_priv) || + IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev)) + bdw_get_stolen_reserved(dev_priv, &reserved_base, + &reserved_size); + else + gen8_get_stolen_reserved(dev_priv, &reserved_base, + &reserved_size); + break; } - if (WARN_ON(bios_reserved > dev_priv->gtt.stolen_size)) + /* It is possible for the reserved base to be zero, but the register + * field for size doesn't have a zero option. */ + if (reserved_base == 0) { + reserved_size = 0; + reserved_base = stolen_top; + } + + if (reserved_base < dev_priv->mm.stolen_base || + reserved_base + reserved_size > stolen_top) { + DRM_DEBUG_KMS("Stolen reserved area [0x%08lx - 0x%08lx] outside stolen memory [0x%08lx - 0x%08lx]\n", + reserved_base, reserved_base + reserved_size, + dev_priv->mm.stolen_base, stolen_top); return 0; + } + + /* It is possible for the reserved area to end before the end of stolen + * memory, so just consider the start. */ + reserved_total = stolen_top - reserved_base; - /* Basic memrange allocator for stolen space */ - drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_size - - bios_reserved); + DRM_DEBUG_KMS("Memory reserved for graphics device: %zuK, usable: %luK\n", + dev_priv->gtt.stolen_size >> 10, + (dev_priv->gtt.stolen_size - reserved_total) >> 10); + + dev_priv->gtt.stolen_usable_size = dev_priv->gtt.stolen_size - + reserved_total; + + /* + * Basic memrange allocator for stolen space. + * + * TODO: Notice that some platforms require us to not use the first page + * of the stolen memory but their BIOSes may still put the framebuffer + * on the first page. So we don't reserve this page for now because of + * that. Our current solution is to just prevent new nodes from being + * inserted on the first page - see the check we have at + * i915_gem_stolen_insert_node_in_range(). We may want to fix the fbcon + * problem later. + */ + drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_usable_size); return 0; } @@ -386,8 +537,10 @@ static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj) static void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + if (obj->stolen) { - drm_mm_remove_node(obj->stolen); + i915_gem_stolen_remove_node(dev_priv, obj->stolen); kfree(obj->stolen); obj->stolen = NULL; } @@ -448,8 +601,7 @@ i915_gem_object_create_stolen(struct drm_device *dev, u32 size) if (!stolen) return NULL; - ret = drm_mm_insert_node(&dev_priv->mm.stolen, stolen, size, - 4096, DRM_MM_SEARCH_DEFAULT); + ret = i915_gem_stolen_insert_node(dev_priv, stolen, size, 4096); if (ret) { kfree(stolen); return NULL; @@ -459,7 +611,7 @@ i915_gem_object_create_stolen(struct drm_device *dev, u32 size) if (obj) return obj; - drm_mm_remove_node(stolen); + i915_gem_stolen_remove_node(dev_priv, stolen); kfree(stolen); return NULL; } @@ -494,7 +646,9 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, stolen->start = stolen_offset; stolen->size = size; + mutex_lock(&dev_priv->mm.stolen_lock); ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen); + mutex_unlock(&dev_priv->mm.stolen_lock); if (ret) { DRM_DEBUG_KMS("failed to allocate stolen space\n"); kfree(stolen); @@ -504,7 +658,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, obj = _i915_gem_object_create_stolen(dev, stolen); if (obj == NULL) { DRM_DEBUG_KMS("failed to allocate stolen object\n"); - drm_mm_remove_node(stolen); + i915_gem_stolen_remove_node(dev_priv, stolen); kfree(stolen); return NULL; } @@ -516,7 +670,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, vma = i915_gem_obj_lookup_or_create_vma(obj, ggtt); if (IS_ERR(vma)) { ret = PTR_ERR(vma); - goto err_out; + goto err; } /* To simplify the initialisation sequence between KMS and GTT, @@ -530,23 +684,19 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, ret = drm_mm_reserve_node(&ggtt->mm, &vma->node); if (ret) { DRM_DEBUG_KMS("failed to allocate stolen GTT space\n"); - goto err_vma; + goto err; } - } - vma->bound |= GLOBAL_BIND; + vma->bound |= GLOBAL_BIND; + list_add_tail(&vma->mm_list, &ggtt->inactive_list); + } list_add_tail(&obj->global_list, &dev_priv->mm.bound_list); - list_add_tail(&vma->mm_list, &ggtt->inactive_list); i915_gem_object_pin_pages(obj); return obj; -err_vma: - i915_gem_vma_destroy(vma); -err_out: - drm_mm_remove_node(stolen); - kfree(stolen); +err: drm_gem_object_unreference(&obj->base); return NULL; } diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index d19c9db5e18c..7410f6c962e7 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -31,201 +31,32 @@ #include <drm/i915_drm.h> #include "i915_drv.h" -/** @file i915_gem_tiling.c - * - * Support for managing tiling state of buffer objects. - * - * The idea behind tiling is to increase cache hit rates by rearranging - * pixel data so that a group of pixel accesses are in the same cacheline. - * Performance improvement from doing this on the back/depth buffer are on - * the order of 30%. - * - * Intel architectures make this somewhat more complicated, though, by - * adjustments made to addressing of data when the memory is in interleaved - * mode (matched pairs of DIMMS) to improve memory bandwidth. - * For interleaved memory, the CPU sends every sequential 64 bytes - * to an alternate memory channel so it can get the bandwidth from both. - * - * The GPU also rearranges its accesses for increased bandwidth to interleaved - * memory, and it matches what the CPU does for non-tiled. However, when tiled - * it does it a little differently, since one walks addresses not just in the - * X direction but also Y. So, along with alternating channels when bit - * 6 of the address flips, it also alternates when other bits flip -- Bits 9 - * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines) - * are common to both the 915 and 965-class hardware. - * - * The CPU also sometimes XORs in higher bits as well, to improve - * bandwidth doing strided access like we do so frequently in graphics. This - * is called "Channel XOR Randomization" in the MCH documentation. The result - * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address - * decode. +/** + * DOC: buffer object tiling * - * All of this bit 6 XORing has an effect on our memory management, - * as we need to make sure that the 3d driver can correctly address object - * contents. + * i915_gem_set_tiling() and i915_gem_get_tiling() is the userspace interface to + * declare fence register requirements. * - * If we don't have interleaved memory, all tiling is safe and no swizzling is - * required. + * In principle GEM doesn't care at all about the internal data layout of an + * object, and hence it also doesn't care about tiling or swizzling. There's two + * exceptions: * - * When bit 17 is XORed in, we simply refuse to tile at all. Bit - * 17 is not just a page offset, so as we page an objet out and back in, - * individual pages in it will have different bit 17 addresses, resulting in - * each 64 bytes being swapped with its neighbor! + * - For X and Y tiling the hardware provides detilers for CPU access, so called + * fences. Since there's only a limited amount of them the kernel must manage + * these, and therefore userspace must tell the kernel the object tiling if it + * wants to use fences for detiling. + * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which + * depends upon the physical page frame number. When swapping such objects the + * page frame number might change and the kernel must be able to fix this up + * and hence now the tiling. Note that on a subset of platforms with + * asymmetric memory channel population the swizzling pattern changes in an + * unknown way, and for those the kernel simply forbids swapping completely. * - * Otherwise, if interleaved, we have to tell the 3d driver what the address - * swizzling it needs to do is, since it's writing with the CPU to the pages - * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the - * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling - * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order - * to match what the GPU expects. - */ - -/** - * Detects bit 6 swizzling of address lookup between IGD access and CPU - * access through main memory. + * Since neither of this applies for new tiling layouts on modern platforms like + * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled. + * Anything else can be handled in userspace entirely without the kernel's + * invovlement. */ -void -i915_gem_detect_bit_6_swizzle(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; - uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; - - if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) { - /* - * On BDW+, swizzling is not used. We leave the CPU memory - * controller in charge of optimizing memory accesses without - * the extra address manipulation GPU side. - * - * VLV and CHV don't have GPU swizzling. - */ - swizzle_x = I915_BIT_6_SWIZZLE_NONE; - swizzle_y = I915_BIT_6_SWIZZLE_NONE; - } else if (INTEL_INFO(dev)->gen >= 6) { - if (dev_priv->preserve_bios_swizzle) { - if (I915_READ(DISP_ARB_CTL) & - DISP_TILE_SURFACE_SWIZZLING) { - swizzle_x = I915_BIT_6_SWIZZLE_9_10; - swizzle_y = I915_BIT_6_SWIZZLE_9; - } else { - swizzle_x = I915_BIT_6_SWIZZLE_NONE; - swizzle_y = I915_BIT_6_SWIZZLE_NONE; - } - } else { - uint32_t dimm_c0, dimm_c1; - dimm_c0 = I915_READ(MAD_DIMM_C0); - dimm_c1 = I915_READ(MAD_DIMM_C1); - dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; - dimm_c1 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; - /* Enable swizzling when the channels are populated - * with identically sized dimms. We don't need to check - * the 3rd channel because no cpu with gpu attached - * ships in that configuration. Also, swizzling only - * makes sense for 2 channels anyway. */ - if (dimm_c0 == dimm_c1) { - swizzle_x = I915_BIT_6_SWIZZLE_9_10; - swizzle_y = I915_BIT_6_SWIZZLE_9; - } else { - swizzle_x = I915_BIT_6_SWIZZLE_NONE; - swizzle_y = I915_BIT_6_SWIZZLE_NONE; - } - } - } else if (IS_GEN5(dev)) { - /* On Ironlake whatever DRAM config, GPU always do - * same swizzling setup. - */ - swizzle_x = I915_BIT_6_SWIZZLE_9_10; - swizzle_y = I915_BIT_6_SWIZZLE_9; - } else if (IS_GEN2(dev)) { - /* As far as we know, the 865 doesn't have these bit 6 - * swizzling issues. - */ - swizzle_x = I915_BIT_6_SWIZZLE_NONE; - swizzle_y = I915_BIT_6_SWIZZLE_NONE; - } else if (IS_MOBILE(dev) || (IS_GEN3(dev) && !IS_G33(dev))) { - uint32_t dcc; - - /* On 9xx chipsets, channel interleave by the CPU is - * determined by DCC. For single-channel, neither the CPU - * nor the GPU do swizzling. For dual channel interleaved, - * the GPU's interleave is bit 9 and 10 for X tiled, and bit - * 9 for Y tiled. The CPU's interleave is independent, and - * can be based on either bit 11 (haven't seen this yet) or - * bit 17 (common). - */ - dcc = I915_READ(DCC); - switch (dcc & DCC_ADDRESSING_MODE_MASK) { - case DCC_ADDRESSING_MODE_SINGLE_CHANNEL: - case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC: - swizzle_x = I915_BIT_6_SWIZZLE_NONE; - swizzle_y = I915_BIT_6_SWIZZLE_NONE; - break; - case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED: - if (dcc & DCC_CHANNEL_XOR_DISABLE) { - /* This is the base swizzling by the GPU for - * tiled buffers. - */ - swizzle_x = I915_BIT_6_SWIZZLE_9_10; - swizzle_y = I915_BIT_6_SWIZZLE_9; - } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) { - /* Bit 11 swizzling by the CPU in addition. */ - swizzle_x = I915_BIT_6_SWIZZLE_9_10_11; - swizzle_y = I915_BIT_6_SWIZZLE_9_11; - } else { - /* Bit 17 swizzling by the CPU in addition. */ - swizzle_x = I915_BIT_6_SWIZZLE_9_10_17; - swizzle_y = I915_BIT_6_SWIZZLE_9_17; - } - break; - } - - /* check for L-shaped memory aka modified enhanced addressing */ - if (IS_GEN4(dev)) { - uint32_t ddc2 = I915_READ(DCC2); - - if (!(ddc2 & DCC2_MODIFIED_ENHANCED_DISABLE)) - dev_priv->quirks |= QUIRK_PIN_SWIZZLED_PAGES; - } - - if (dcc == 0xffffffff) { - DRM_ERROR("Couldn't read from MCHBAR. " - "Disabling tiling.\n"); - swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; - swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; - } - } else { - /* The 965, G33, and newer, have a very flexible memory - * configuration. It will enable dual-channel mode - * (interleaving) on as much memory as it can, and the GPU - * will additionally sometimes enable different bit 6 - * swizzling for tiled objects from the CPU. - * - * Here's what I found on the G965: - * slot fill memory size swizzling - * 0A 0B 1A 1B 1-ch 2-ch - * 512 0 0 0 512 0 O - * 512 0 512 0 16 1008 X - * 512 0 0 512 16 1008 X - * 0 512 0 512 16 1008 X - * 1024 1024 1024 0 2048 1024 O - * - * We could probably detect this based on either the DRB - * matching, which was the case for the swizzling required in - * the table above, or from the 1-ch value being less than - * the minimum size of a rank. - */ - if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) { - swizzle_x = I915_BIT_6_SWIZZLE_NONE; - swizzle_y = I915_BIT_6_SWIZZLE_NONE; - } else { - swizzle_x = I915_BIT_6_SWIZZLE_9_10; - swizzle_y = I915_BIT_6_SWIZZLE_9; - } - } - - dev_priv->mm.bit_6_swizzle_x = swizzle_x; - dev_priv->mm.bit_6_swizzle_y = swizzle_y; -} /* Check pitch constriants for all chips & tiling formats */ static bool @@ -313,8 +144,18 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode) } /** + * i915_gem_set_tiling - IOCTL handler to set tiling mode + * @dev: DRM device + * @data: data pointer for the ioctl + * @file: DRM file for the ioctl call + * * Sets the tiling mode of an object, returning the required swizzling of * bit 6 of addresses in the object. + * + * Called by the user via ioctl. + * + * Returns: + * Zero on success, negative errno on failure. */ int i915_gem_set_tiling(struct drm_device *dev, void *data, @@ -335,6 +176,8 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, return -EINVAL; } + intel_runtime_pm_get(dev_priv); + mutex_lock(&dev->struct_mutex); if (obj->pin_display || obj->framebuffer_references) { ret = -EBUSY; @@ -428,11 +271,23 @@ err: drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); + intel_runtime_pm_put(dev_priv); + return ret; } /** + * i915_gem_get_tiling - IOCTL handler to get tiling mode + * @dev: DRM device + * @data: data pointer for the ioctl + * @file: DRM file for the ioctl call + * * Returns the current tiling mode and required bit 6 swizzling for the object. + * + * Called by the user via ioctl. + * + * Returns: + * Zero on success, negative errno on failure. */ int i915_gem_get_tiling(struct drm_device *dev, void *data, @@ -478,75 +333,3 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, return 0; } - -/** - * Swap every 64 bytes of this page around, to account for it having a new - * bit 17 of its physical address and therefore being interpreted differently - * by the GPU. - */ -static void -i915_gem_swizzle_page(struct page *page) -{ - char temp[64]; - char *vaddr; - int i; - - vaddr = kmap(page); - - for (i = 0; i < PAGE_SIZE; i += 128) { - memcpy(temp, &vaddr[i], 64); - memcpy(&vaddr[i], &vaddr[i + 64], 64); - memcpy(&vaddr[i + 64], temp, 64); - } - - kunmap(page); -} - -void -i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) -{ - struct sg_page_iter sg_iter; - int i; - - if (obj->bit_17 == NULL) - return; - - i = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - struct page *page = sg_page_iter_page(&sg_iter); - char new_bit_17 = page_to_phys(page) >> 17; - if ((new_bit_17 & 0x1) != - (test_bit(i, obj->bit_17) != 0)) { - i915_gem_swizzle_page(page); - set_page_dirty(page); - } - i++; - } -} - -void -i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) -{ - struct sg_page_iter sg_iter; - int page_count = obj->base.size >> PAGE_SHIFT; - int i; - - if (obj->bit_17 == NULL) { - obj->bit_17 = kcalloc(BITS_TO_LONGS(page_count), - sizeof(long), GFP_KERNEL); - if (obj->bit_17 == NULL) { - DRM_ERROR("Failed to allocate memory for bit 17 " - "record\n"); - return; - } - } - - i = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - if (page_to_phys(sg_page_iter_page(&sg_iter)) & (1 << 17)) - __set_bit(i, obj->bit_17); - else - __clear_bit(i, obj->bit_17); - i++; - } -} diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index 8fd431bcdfd3..19fb0bddc1cd 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -50,7 +50,6 @@ struct i915_mmu_notifier { struct mmu_notifier mn; struct rb_root objects; struct list_head linear; - unsigned long serial; bool has_linear; }; @@ -59,13 +58,16 @@ struct i915_mmu_object { struct interval_tree_node it; struct list_head link; struct drm_i915_gem_object *obj; + struct work_struct work; + bool active; bool is_linear; }; -static unsigned long cancel_userptr(struct drm_i915_gem_object *obj) +static void __cancel_userptr__worker(struct work_struct *work) { + struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); + struct drm_i915_gem_object *obj = mo->obj; struct drm_device *dev = obj->base.dev; - unsigned long end; mutex_lock(&dev->struct_mutex); /* Cancel any active worker and force us to re-evaluate gup */ @@ -88,45 +90,28 @@ static unsigned long cancel_userptr(struct drm_i915_gem_object *obj) dev_priv->mm.interruptible = was_interruptible; } - end = obj->userptr.ptr + obj->base.size; - drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); - - return end; } -static void *invalidate_range__linear(struct i915_mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, - unsigned long end) +static unsigned long cancel_userptr(struct i915_mmu_object *mo) { - struct i915_mmu_object *mo; - unsigned long serial; - -restart: - serial = mn->serial; - list_for_each_entry(mo, &mn->linear, link) { - struct drm_i915_gem_object *obj; - - if (mo->it.last < start || mo->it.start > end) - continue; - - obj = mo->obj; - - if (!kref_get_unless_zero(&obj->base.refcount)) - continue; - - spin_unlock(&mn->lock); - - cancel_userptr(obj); - - spin_lock(&mn->lock); - if (serial != mn->serial) - goto restart; + unsigned long end = mo->obj->userptr.ptr + mo->obj->base.size; + + /* The mmu_object is released late when destroying the + * GEM object so it is entirely possible to gain a + * reference on an object in the process of being freed + * since our serialisation is via the spinlock and not + * the struct_mutex - and consequently use it after it + * is freed and then double free it. + */ + if (mo->active && kref_get_unless_zero(&mo->obj->base.refcount)) { + schedule_work(&mo->work); + /* only schedule one work packet to avoid the refleak */ + mo->active = false; } - return NULL; + return end; } static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, @@ -134,46 +119,32 @@ static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, unsigned long start, unsigned long end) { - struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn); - struct interval_tree_node *it = NULL; - unsigned long next = start; - unsigned long serial = 0; - - end--; /* interval ranges are inclusive, but invalidate range is exclusive */ - while (next < end) { - struct drm_i915_gem_object *obj = NULL; - - spin_lock(&mn->lock); - if (mn->has_linear) - it = invalidate_range__linear(mn, mm, start, end); - else if (serial == mn->serial) - it = interval_tree_iter_next(it, next, end); - else - it = interval_tree_iter_first(&mn->objects, start, end); - if (it != NULL) { - obj = container_of(it, struct i915_mmu_object, it)->obj; - - /* The mmu_object is released late when destroying the - * GEM object so it is entirely possible to gain a - * reference on an object in the process of being freed - * since our serialisation is via the spinlock and not - * the struct_mutex - and consequently use it after it - * is freed and then double free it. - */ - if (!kref_get_unless_zero(&obj->base.refcount)) { - spin_unlock(&mn->lock); - serial = 0; + struct i915_mmu_notifier *mn = + container_of(_mn, struct i915_mmu_notifier, mn); + struct i915_mmu_object *mo; + + /* interval ranges are inclusive, but invalidate range is exclusive */ + end--; + + spin_lock(&mn->lock); + if (mn->has_linear) { + list_for_each_entry(mo, &mn->linear, link) { + if (mo->it.last < start || mo->it.start > end) continue; - } - serial = mn->serial; + cancel_userptr(mo); } - spin_unlock(&mn->lock); - if (obj == NULL) - return; + } else { + struct interval_tree_node *it; - next = cancel_userptr(obj); + it = interval_tree_iter_first(&mn->objects, start, end); + while (it) { + mo = container_of(it, struct i915_mmu_object, it); + start = cancel_userptr(mo); + it = interval_tree_iter_next(it, start, end); + } } + spin_unlock(&mn->lock); } static const struct mmu_notifier_ops i915_gem_userptr_notifier = { @@ -193,7 +164,6 @@ i915_mmu_notifier_create(struct mm_struct *mm) spin_lock_init(&mn->lock); mn->mn.ops = &i915_gem_userptr_notifier; mn->objects = RB_ROOT; - mn->serial = 1; INIT_LIST_HEAD(&mn->linear); mn->has_linear = false; @@ -207,12 +177,6 @@ i915_mmu_notifier_create(struct mm_struct *mm) return mn; } -static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mn) -{ - if (++mn->serial == 0) - mn->serial = 1; -} - static int i915_mmu_notifier_add(struct drm_device *dev, struct i915_mmu_notifier *mn, @@ -259,10 +223,9 @@ i915_mmu_notifier_add(struct drm_device *dev, } else interval_tree_insert(&mo->it, &mn->objects); - if (ret == 0) { + if (ret == 0) list_add(&mo->link, &mn->linear); - __i915_mmu_notifier_update_serial(mn); - } + spin_unlock(&mn->lock); mutex_unlock(&dev->struct_mutex); @@ -290,7 +253,6 @@ i915_mmu_notifier_del(struct i915_mmu_notifier *mn, mn->has_linear = i915_mmu_notifier_has_linear(mn); else interval_tree_remove(&mo->it, &mn->objects); - __i915_mmu_notifier_update_serial(mn); spin_unlock(&mn->lock); } @@ -357,6 +319,7 @@ i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, mo->it.start = obj->userptr.ptr; mo->it.last = mo->it.start + obj->base.size - 1; mo->obj = obj; + INIT_WORK(&mo->work, __cancel_userptr__worker); ret = i915_mmu_notifier_add(obj->base.dev, mn, mo); if (ret) { @@ -565,31 +528,65 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, return ret; } +static int +__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, + bool value) +{ + int ret = 0; + + /* During mm_invalidate_range we need to cancel any userptr that + * overlaps the range being invalidated. Doing so requires the + * struct_mutex, and that risks recursion. In order to cause + * recursion, the user must alias the userptr address space with + * a GTT mmapping (possible with a MAP_FIXED) - then when we have + * to invalidate that mmaping, mm_invalidate_range is called with + * the userptr address *and* the struct_mutex held. To prevent that + * we set a flag under the i915_mmu_notifier spinlock to indicate + * whether this object is valid. + */ +#if defined(CONFIG_MMU_NOTIFIER) + if (obj->userptr.mmu_object == NULL) + return 0; + + spin_lock(&obj->userptr.mmu_object->mn->lock); + /* In order to serialise get_pages with an outstanding + * cancel_userptr, we must drop the struct_mutex and try again. + */ + if (!value || !work_pending(&obj->userptr.mmu_object->work)) + obj->userptr.mmu_object->active = value; + else + ret = -EAGAIN; + spin_unlock(&obj->userptr.mmu_object->mn->lock); +#endif + + return ret; +} + static void __i915_gem_userptr_get_pages_worker(struct work_struct *_work) { struct get_pages_work *work = container_of(_work, typeof(*work), work); struct drm_i915_gem_object *obj = work->obj; struct drm_device *dev = obj->base.dev; - const int num_pages = obj->base.size >> PAGE_SHIFT; + const int npages = obj->base.size >> PAGE_SHIFT; struct page **pvec; int pinned, ret; ret = -ENOMEM; pinned = 0; - pvec = kmalloc(num_pages*sizeof(struct page *), + pvec = kmalloc(npages*sizeof(struct page *), GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); if (pvec == NULL) - pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); + pvec = drm_malloc_ab(npages, sizeof(struct page *)); if (pvec != NULL) { struct mm_struct *mm = obj->userptr.mm->mm; down_read(&mm->mmap_sem); - while (pinned < num_pages) { + while (pinned < npages) { ret = get_user_pages(work->task, mm, obj->userptr.ptr + pinned * PAGE_SIZE, - num_pages - pinned, + npages - pinned, !obj->userptr.read_only, 0, pvec + pinned, NULL); if (ret < 0) @@ -601,20 +598,22 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) } mutex_lock(&dev->struct_mutex); - if (obj->userptr.work != &work->work) { - ret = 0; - } else if (pinned == num_pages) { - ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); - if (ret == 0) { - list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - - pinned = 0; + if (obj->userptr.work == &work->work) { + if (pinned == npages) { + ret = __i915_gem_userptr_set_pages(obj, pvec, npages); + if (ret == 0) { + list_add_tail(&obj->global_list, + &to_i915(dev)->mm.unbound_list); + obj->get_page.sg = obj->pages->sgl; + obj->get_page.last = 0; + pinned = 0; + } } + obj->userptr.work = ERR_PTR(ret); + if (ret) + __i915_gem_userptr_set_active(obj, false); } - obj->userptr.work = ERR_PTR(ret); obj->userptr.workers--; drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); @@ -627,11 +626,60 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) } static int +__i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, + bool *active) +{ + struct get_pages_work *work; + + /* Spawn a worker so that we can acquire the + * user pages without holding our mutex. Access + * to the user pages requires mmap_sem, and we have + * a strict lock ordering of mmap_sem, struct_mutex - + * we already hold struct_mutex here and so cannot + * call gup without encountering a lock inversion. + * + * Userspace will keep on repeating the operation + * (thanks to EAGAIN) until either we hit the fast + * path or the worker completes. If the worker is + * cancelled or superseded, the task is still run + * but the results ignored. (This leads to + * complications that we may have a stray object + * refcount that we need to be wary of when + * checking for existing objects during creation.) + * If the worker encounters an error, it reports + * that error back to this function through + * obj->userptr.work = ERR_PTR. + */ + if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS) + return -EAGAIN; + + work = kmalloc(sizeof(*work), GFP_KERNEL); + if (work == NULL) + return -ENOMEM; + + obj->userptr.work = &work->work; + obj->userptr.workers++; + + work->obj = obj; + drm_gem_object_reference(&obj->base); + + work->task = current; + get_task_struct(work->task); + + INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); + schedule_work(&work->work); + + *active = true; + return -EAGAIN; +} + +static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) { const int num_pages = obj->base.size >> PAGE_SHIFT; struct page **pvec; int pinned, ret; + bool active; /* If userspace should engineer that these pages are replaced in * the vma between us binding this page into the GTT and completion @@ -649,6 +697,20 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) * to the vma (discard or cloning) which should prevent the more * egregious cases from causing harm. */ + if (IS_ERR(obj->userptr.work)) { + /* active flag will have been dropped already by the worker */ + ret = PTR_ERR(obj->userptr.work); + obj->userptr.work = NULL; + return ret; + } + if (obj->userptr.work) + /* active flag should still be held for the pending work */ + return -EAGAIN; + + /* Let the mmu-notifier know that we have begun and need cancellation */ + ret = __i915_gem_userptr_set_active(obj, true); + if (ret) + return ret; pvec = NULL; pinned = 0; @@ -657,73 +719,27 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); if (pvec == NULL) { pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); - if (pvec == NULL) + if (pvec == NULL) { + __i915_gem_userptr_set_active(obj, false); return -ENOMEM; + } } pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, !obj->userptr.read_only, pvec); } - if (pinned < num_pages) { - if (pinned < 0) { - ret = pinned; - pinned = 0; - } else { - /* Spawn a worker so that we can acquire the - * user pages without holding our mutex. Access - * to the user pages requires mmap_sem, and we have - * a strict lock ordering of mmap_sem, struct_mutex - - * we already hold struct_mutex here and so cannot - * call gup without encountering a lock inversion. - * - * Userspace will keep on repeating the operation - * (thanks to EAGAIN) until either we hit the fast - * path or the worker completes. If the worker is - * cancelled or superseded, the task is still run - * but the results ignored. (This leads to - * complications that we may have a stray object - * refcount that we need to be wary of when - * checking for existing objects during creation.) - * If the worker encounters an error, it reports - * that error back to this function through - * obj->userptr.work = ERR_PTR. - */ - ret = -EAGAIN; - if (obj->userptr.work == NULL && - obj->userptr.workers < I915_GEM_USERPTR_MAX_WORKERS) { - struct get_pages_work *work; - - work = kmalloc(sizeof(*work), GFP_KERNEL); - if (work != NULL) { - obj->userptr.work = &work->work; - obj->userptr.workers++; - - work->obj = obj; - drm_gem_object_reference(&obj->base); - - work->task = current; - get_task_struct(work->task); - - INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); - schedule_work(&work->work); - } else - ret = -ENOMEM; - } else { - if (IS_ERR(obj->userptr.work)) { - ret = PTR_ERR(obj->userptr.work); - obj->userptr.work = NULL; - } - } - } - } else { + + active = false; + if (pinned < 0) + ret = pinned, pinned = 0; + else if (pinned < num_pages) + ret = __i915_gem_userptr_get_pages_schedule(obj, &active); + else ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); - if (ret == 0) { - obj->userptr.work = NULL; - pinned = 0; - } + if (ret) { + __i915_gem_userptr_set_active(obj, active); + release_pages(pvec, pinned, 0); } - - release_pages(pvec, pinned, 0); drm_free_large(pvec); return ret; } @@ -734,6 +750,7 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) struct sg_page_iter sg_iter; BUG_ON(obj->userptr.work != NULL); + __i915_gem_userptr_set_active(obj, false); if (obj->madv != I915_MADV_WILLNEED) obj->dirty = 0; @@ -804,7 +821,10 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { * Also note, that the object created here is not currently a "first class" * object, in that several ioctls are banned. These are the CPU access * ioctls: mmap(), pwrite and pread. In practice, you are expected to use - * direct access via your pointer rather than use those ioctls. + * direct access via your pointer rather than use those ioctls. Another + * restriction is that we do not allow userptr surfaces to be pinned to the + * hardware and so we reject any attempt to create a framebuffer out of a + * userptr. * * If you think this is a good interface to use to pass GPU memory between * drivers, please use dma-buf instead. In fact, wherever possible use @@ -813,7 +833,6 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_userptr *args = data; struct drm_i915_gem_object *obj; int ret; @@ -826,9 +845,6 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file if (offset_in_page(args->user_ptr | args->user_size)) return -EINVAL; - if (args->user_size > dev_priv->gtt.base.total) - return -E2BIG; - if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, (char __user *)(unsigned long)args->user_ptr, args->user_size)) return -EFAULT; diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 6f4256918f76..06ca4082735b 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -30,11 +30,6 @@ #include <generated/utsrelease.h> #include "i915_drv.h" -static const char *yesno(int v) -{ - return v ? "yes" : "no"; -} - static const char *ring_str(int ring) { switch (ring) { @@ -197,8 +192,9 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m, err_printf(m, " %s [%d]:\n", name, count); while (count--) { - err_printf(m, " %08x %8u %02x %02x [ ", - err->gtt_offset, + err_printf(m, " %08x_%08x %8u %02x %02x [ ", + upper_32_bits(err->gtt_offset), + lower_32_bits(err->gtt_offset), err->size, err->read_domains, err->write_domain); @@ -369,6 +365,18 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, "Reset count: %u\n", error->reset_count); err_printf(m, "Suspend count: %u\n", error->suspend_count); err_printf(m, "PCI ID: 0x%04x\n", dev->pdev->device); + err_printf(m, "IOMMU enabled?: %d\n", error->iommu); + + if (HAS_CSR(dev)) { + struct intel_csr *csr = &dev_priv->csr; + + err_printf(m, "DMC loaded: %s\n", + yesno(csr->dmc_payload != NULL)); + err_printf(m, "DMC fw version: %d.%d\n", + CSR_VERSION_MAJOR(csr->version), + CSR_VERSION_MINOR(csr->version)); + } + err_printf(m, "EIR: 0x%08x\n", error->eir); err_printf(m, "IER: 0x%08x\n", error->ier); if (INTEL_INFO(dev)->gen >= 8) { @@ -426,15 +434,17 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, " (submitted by %s [%d])", error->ring[i].comm, error->ring[i].pid); - err_printf(m, " --- gtt_offset = 0x%08x\n", - obj->gtt_offset); + err_printf(m, " --- gtt_offset = 0x%08x %08x\n", + upper_32_bits(obj->gtt_offset), + lower_32_bits(obj->gtt_offset)); print_error_obj(m, obj); } obj = error->ring[i].wa_batchbuffer; if (obj) { err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n", - dev_priv->ring[i].name, obj->gtt_offset); + dev_priv->ring[i].name, + lower_32_bits(obj->gtt_offset)); print_error_obj(m, obj); } @@ -453,22 +463,28 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, if ((obj = error->ring[i].ringbuffer)) { err_printf(m, "%s --- ringbuffer = 0x%08x\n", dev_priv->ring[i].name, - obj->gtt_offset); + lower_32_bits(obj->gtt_offset)); print_error_obj(m, obj); } if ((obj = error->ring[i].hws_page)) { - err_printf(m, "%s --- HW Status = 0x%08x\n", - dev_priv->ring[i].name, - obj->gtt_offset); + u64 hws_offset = obj->gtt_offset; + u32 *hws_page = &obj->pages[0][0]; + + if (i915.enable_execlists) { + hws_offset += LRC_PPHWSP_PN * PAGE_SIZE; + hws_page = &obj->pages[LRC_PPHWSP_PN][0]; + } + err_printf(m, "%s --- HW Status = 0x%08llx\n", + dev_priv->ring[i].name, hws_offset); offset = 0; for (elt = 0; elt < PAGE_SIZE/16; elt += 4) { err_printf(m, "[%04x] %08x %08x %08x %08x\n", offset, - obj->pages[0][elt], - obj->pages[0][elt+1], - obj->pages[0][elt+2], - obj->pages[0][elt+3]); + hws_page[elt], + hws_page[elt+1], + hws_page[elt+2], + hws_page[elt+3]); offset += 16; } } @@ -476,13 +492,14 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, if ((obj = error->ring[i].ctx)) { err_printf(m, "%s --- HW Context = 0x%08x\n", dev_priv->ring[i].name, - obj->gtt_offset); + lower_32_bits(obj->gtt_offset)); print_error_obj(m, obj); } } if ((obj = error->semaphore_obj)) { - err_printf(m, "Semaphore page = 0x%08x\n", obj->gtt_offset); + err_printf(m, "Semaphore page = 0x%08x\n", + lower_32_bits(obj->gtt_offset)); for (elt = 0; elt < PAGE_SIZE/16; elt += 4) { err_printf(m, "[%04x] %08x %08x %08x %08x\n", elt * 4, @@ -590,7 +607,7 @@ i915_error_object_create(struct drm_i915_private *dev_priv, int num_pages; bool use_ggtt; int i = 0; - u32 reloc_offset; + u64 reloc_offset; if (src == NULL || src->pages == NULL) return NULL; @@ -786,20 +803,15 @@ static void i915_gem_record_fences(struct drm_device *dev, int i; if (IS_GEN3(dev) || IS_GEN2(dev)) { - for (i = 0; i < 8; i++) - error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4)); - if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) - for (i = 0; i < 8; i++) - error->fence[i+8] = I915_READ(FENCE_REG_945_8 + - (i * 4)); - } else if (IS_GEN5(dev) || IS_GEN4(dev)) - for (i = 0; i < 16; i++) - error->fence[i] = I915_READ64(FENCE_REG_965_0 + - (i * 8)); - else if (INTEL_INFO(dev)->gen >= 6) for (i = 0; i < dev_priv->num_fence_regs; i++) - error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + - (i * 8)); + error->fence[i] = I915_READ(FENCE_REG(i)); + } else if (IS_GEN5(dev) || IS_GEN4(dev)) { + for (i = 0; i < dev_priv->num_fence_regs; i++) + error->fence[i] = I915_READ64(FENCE_REG_965_LO(i)); + } else if (INTEL_INFO(dev)->gen >= 6) { + for (i = 0; i < dev_priv->num_fence_regs; i++) + error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i)); + } } @@ -861,7 +873,7 @@ static void i915_record_ring_state(struct drm_device *dev, struct drm_i915_private *dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen >= 6) { - ering->rc_psmi = I915_READ(ring->mmio_base + 0x50); + ering->rc_psmi = I915_READ(RING_PSMI_CTL(ring->mmio_base)); ering->fault_reg = I915_READ(RING_FAULT_REG(ring)); if (INTEL_INFO(dev)->gen >= 8) gen8_record_semaphore_state(dev_priv, error, ring, ering); @@ -885,7 +897,7 @@ static void i915_record_ring_state(struct drm_device *dev, ering->faddr = I915_READ(DMA_FADD_I8XX); ering->ipeir = I915_READ(IPEIR); ering->ipehr = I915_READ(IPEHR); - ering->instdone = I915_READ(INSTDONE); + ering->instdone = I915_READ(GEN2_INSTDONE); } ering->waiting = waitqueue_active(&ring->irq_queue); @@ -898,7 +910,7 @@ static void i915_record_ring_state(struct drm_device *dev, ering->ctl = I915_READ_CTL(ring); if (I915_NEED_GFX_HWS(dev)) { - int mmio; + i915_reg_t mmio; if (IS_GEN7(dev)) { switch (ring->id) { @@ -1070,6 +1082,25 @@ static void i915_gem_record_rings(struct drm_device *dev, list_for_each_entry(request, &ring->request_list, list) { struct drm_i915_error_request *erq; + if (count >= error->ring[i].num_requests) { + /* + * If the ring request list was changed in + * between the point where the error request + * list was created and dimensioned and this + * point then just exit early to avoid crashes. + * + * We don't need to communicate that the + * request list changed state during error + * state capture and that the error state is + * slightly incorrect as a consequence since we + * are typically only interested in the request + * list state at the point of error state + * capture, not in any changes happening during + * the capture. + */ + break; + } + erq = &error->ring[i].requests[count++]; erq->seqno = request->seqno; erq->jiffies = request->emitted_jiffies; @@ -1180,7 +1211,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, if (IS_VALLEYVIEW(dev)) { error->gtier[0] = I915_READ(GTIER); error->ier = I915_READ(VLV_IER); - error->forcewake = I915_READ(FORCEWAKE_VLV); + error->forcewake = I915_READ_FW(FORCEWAKE_VLV); } if (IS_GEN7(dev)) @@ -1192,14 +1223,14 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, } if (IS_GEN6(dev)) { - error->forcewake = I915_READ(FORCEWAKE); + error->forcewake = I915_READ_FW(FORCEWAKE); error->gab_ctl = I915_READ(GAB_CTL); error->gfx_mode = I915_READ(GFX_MODE); } /* 2: Registers which belong to multiple generations */ if (INTEL_INFO(dev)->gen >= 7) - error->forcewake = I915_READ(FORCEWAKE_MT); + error->forcewake = I915_READ_FW(FORCEWAKE_MT); if (INTEL_INFO(dev)->gen >= 6) { error->derrmr = I915_READ(DERRMR); @@ -1266,6 +1297,10 @@ static void i915_error_capture_msg(struct drm_device *dev, static void i915_capture_gen_state(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error) { + error->iommu = -1; +#ifdef CONFIG_INTEL_IOMMU + error->iommu = intel_iommu_gfx_mapped; +#endif error->reset_count = i915_reset_count(&dev_priv->gpu_error); error->suspend_count = dev_priv->suspend_count; } @@ -1383,12 +1418,12 @@ void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone) memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG); if (IS_GEN2(dev) || IS_GEN3(dev)) - instdone[0] = I915_READ(INSTDONE); + instdone[0] = I915_READ(GEN2_INSTDONE); else if (IS_GEN4(dev) || IS_GEN5(dev) || IS_GEN6(dev)) { - instdone[0] = I915_READ(INSTDONE_I965); - instdone[1] = I915_READ(INSTDONE1); + instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE)); + instdone[1] = I915_READ(GEN4_INSTDONE1); } else if (INTEL_INFO(dev)->gen >= 7) { - instdone[0] = I915_READ(GEN7_INSTDONE_1); + instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE)); instdone[1] = I915_READ(GEN7_SC_INSTDONE); instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE); instdone[3] = I915_READ(GEN7_ROW_INSTDONE); diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/i915_guc_reg.h new file mode 100644 index 000000000000..685c7991e24f --- /dev/null +++ b/drivers/gpu/drm/i915/i915_guc_reg.h @@ -0,0 +1,110 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ +#ifndef _I915_GUC_REG_H_ +#define _I915_GUC_REG_H_ + +/* Definitions of GuC H/W registers, bits, etc */ + +#define GUC_STATUS _MMIO(0xc000) +#define GS_BOOTROM_SHIFT 1 +#define GS_BOOTROM_MASK (0x7F << GS_BOOTROM_SHIFT) +#define GS_BOOTROM_RSA_FAILED (0x50 << GS_BOOTROM_SHIFT) +#define GS_UKERNEL_SHIFT 8 +#define GS_UKERNEL_MASK (0xFF << GS_UKERNEL_SHIFT) +#define GS_UKERNEL_LAPIC_DONE (0x30 << GS_UKERNEL_SHIFT) +#define GS_UKERNEL_DPC_ERROR (0x60 << GS_UKERNEL_SHIFT) +#define GS_UKERNEL_READY (0xF0 << GS_UKERNEL_SHIFT) +#define GS_MIA_SHIFT 16 +#define GS_MIA_MASK (0x07 << GS_MIA_SHIFT) +#define GS_MIA_CORE_STATE (1 << GS_MIA_SHIFT) + +#define SOFT_SCRATCH(n) _MMIO(0xc180 + (n) * 4) + +#define UOS_RSA_SCRATCH(i) _MMIO(0xc200 + (i) * 4) +#define UOS_RSA_SCRATCH_MAX_COUNT 64 +#define DMA_ADDR_0_LOW _MMIO(0xc300) +#define DMA_ADDR_0_HIGH _MMIO(0xc304) +#define DMA_ADDR_1_LOW _MMIO(0xc308) +#define DMA_ADDR_1_HIGH _MMIO(0xc30c) +#define DMA_ADDRESS_SPACE_WOPCM (7 << 16) +#define DMA_ADDRESS_SPACE_GTT (8 << 16) +#define DMA_COPY_SIZE _MMIO(0xc310) +#define DMA_CTRL _MMIO(0xc314) +#define UOS_MOVE (1<<4) +#define START_DMA (1<<0) +#define DMA_GUC_WOPCM_OFFSET _MMIO(0xc340) +#define GUC_WOPCM_OFFSET_VALUE 0x80000 /* 512KB */ +#define GUC_MAX_IDLE_COUNT _MMIO(0xC3E4) + +#define GUC_WOPCM_SIZE _MMIO(0xc050) +#define GUC_WOPCM_SIZE_VALUE (0x80 << 12) /* 512KB */ + +/* GuC addresses below GUC_WOPCM_TOP don't map through the GTT */ +#define GUC_WOPCM_TOP (GUC_WOPCM_SIZE_VALUE) + +#define GEN8_GT_PM_CONFIG _MMIO(0x138140) +#define GEN9LP_GT_PM_CONFIG _MMIO(0x138140) +#define GEN9_GT_PM_CONFIG _MMIO(0x13816c) +#define GT_DOORBELL_ENABLE (1<<0) + +#define GEN8_GTCR _MMIO(0x4274) +#define GEN8_GTCR_INVALIDATE (1<<0) + +#define GUC_ARAT_C6DIS _MMIO(0xA178) + +#define GUC_SHIM_CONTROL _MMIO(0xc064) +#define GUC_DISABLE_SRAM_INIT_TO_ZEROES (1<<0) +#define GUC_ENABLE_READ_CACHE_LOGIC (1<<1) +#define GUC_ENABLE_MIA_CACHING (1<<2) +#define GUC_GEN10_MSGCH_ENABLE (1<<4) +#define GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA (1<<9) +#define GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA (1<<10) +#define GUC_ENABLE_MIA_CLOCK_GATING (1<<15) +#define GUC_GEN10_SHIM_WC_ENABLE (1<<21) + +#define GUC_SHIM_CONTROL_VALUE (GUC_DISABLE_SRAM_INIT_TO_ZEROES | \ + GUC_ENABLE_READ_CACHE_LOGIC | \ + GUC_ENABLE_MIA_CACHING | \ + GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA | \ + GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA | \ + GUC_ENABLE_MIA_CLOCK_GATING) + +#define HOST2GUC_INTERRUPT _MMIO(0xc4c8) +#define HOST2GUC_TRIGGER (1<<0) + +#define DRBMISC1 0x1984 +#define DOORBELL_ENABLE (1<<0) + +#define GEN8_DRBREGL(x) _MMIO(0x1000 + (x) * 8) +#define GEN8_DRB_VALID (1<<0) +#define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) + +#define DE_GUCRMR _MMIO(0x44054) + +#define GUC_BCS_RCS_IER _MMIO(0xC550) +#define GUC_VCS2_VCS1_IER _MMIO(0xC554) +#define GUC_WD_VECS_IER _MMIO(0xC558) +#define GUC_PM_P24C_IER _MMIO(0xC55C) + +#endif diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c new file mode 100644 index 000000000000..0d23785ba818 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -0,0 +1,956 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ +#include <linux/firmware.h> +#include <linux/circ_buf.h> +#include "i915_drv.h" +#include "intel_guc.h" + +/** + * DOC: GuC-based command submission + * + * i915_guc_client: + * We use the term client to avoid confusion with contexts. A i915_guc_client is + * equivalent to GuC object guc_context_desc. This context descriptor is + * allocated from a pool of 1024 entries. Kernel driver will allocate doorbell + * and workqueue for it. Also the process descriptor (guc_process_desc), which + * is mapped to client space. So the client can write Work Item then ring the + * doorbell. + * + * To simplify the implementation, we allocate one gem object that contains all + * pages for doorbell, process descriptor and workqueue. + * + * The Scratch registers: + * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes + * a value to the action register (SOFT_SCRATCH_0) along with any data. It then + * triggers an interrupt on the GuC via another register write (0xC4C8). + * Firmware writes a success/fail code back to the action register after + * processes the request. The kernel driver polls waiting for this update and + * then proceeds. + * See host2guc_action() + * + * Doorbells: + * Doorbells are interrupts to uKernel. A doorbell is a single cache line (QW) + * mapped into process space. + * + * Work Items: + * There are several types of work items that the host may place into a + * workqueue, each with its own requirements and limitations. Currently only + * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which + * represents in-order queue. The kernel driver packs ring tail pointer and an + * ELSP context descriptor dword into Work Item. + * See guc_add_workqueue_item() + * + */ + +/* + * Read GuC command/status register (SOFT_SCRATCH_0) + * Return true if it contains a response rather than a command + */ +static inline bool host2guc_action_response(struct drm_i915_private *dev_priv, + u32 *status) +{ + u32 val = I915_READ(SOFT_SCRATCH(0)); + *status = val; + return GUC2HOST_IS_RESPONSE(val); +} + +static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + u32 status; + int i; + int ret; + + if (WARN_ON(len < 1 || len > 15)) + return -EINVAL; + + intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); + + dev_priv->guc.action_count += 1; + dev_priv->guc.action_cmd = data[0]; + + for (i = 0; i < len; i++) + I915_WRITE(SOFT_SCRATCH(i), data[i]); + + POSTING_READ(SOFT_SCRATCH(i - 1)); + + I915_WRITE(HOST2GUC_INTERRUPT, HOST2GUC_TRIGGER); + + /* No HOST2GUC command should take longer than 10ms */ + ret = wait_for_atomic(host2guc_action_response(dev_priv, &status), 10); + if (status != GUC2HOST_STATUS_SUCCESS) { + /* + * Either the GuC explicitly returned an error (which + * we convert to -EIO here) or no response at all was + * received within the timeout limit (-ETIMEDOUT) + */ + if (ret != -ETIMEDOUT) + ret = -EIO; + + DRM_ERROR("GUC: host2guc action 0x%X failed. ret=%d " + "status=0x%08X response=0x%08X\n", + data[0], ret, status, + I915_READ(SOFT_SCRATCH(15))); + + dev_priv->guc.action_fail += 1; + dev_priv->guc.action_err = ret; + } + dev_priv->guc.action_status = status; + + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + + return ret; +} + +/* + * Tell the GuC to allocate or deallocate a specific doorbell + */ + +static int host2guc_allocate_doorbell(struct intel_guc *guc, + struct i915_guc_client *client) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_ALLOCATE_DOORBELL; + data[1] = client->ctx_index; + + return host2guc_action(guc, data, 2); +} + +static int host2guc_release_doorbell(struct intel_guc *guc, + struct i915_guc_client *client) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_DEALLOCATE_DOORBELL; + data[1] = client->ctx_index; + + return host2guc_action(guc, data, 2); +} + +static int host2guc_sample_forcewake(struct intel_guc *guc, + struct i915_guc_client *client) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct drm_device *dev = dev_priv->dev; + u32 data[2]; + + data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE; + /* WaRsDisableCoarsePowerGating:skl,bxt */ + if (!intel_enable_rc6(dev_priv->dev) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1) || + (IS_SKL_GT3(dev) && IS_SKL_REVID(dev, 0, SKL_REVID_E0)) || + (IS_SKL_GT4(dev) && IS_SKL_REVID(dev, 0, SKL_REVID_E0))) + data[1] = 0; + else + /* bit 0 and 1 are for Render and Media domain separately */ + data[1] = GUC_FORCEWAKE_RENDER | GUC_FORCEWAKE_MEDIA; + + return host2guc_action(guc, data, ARRAY_SIZE(data)); +} + +/* + * Initialise, update, or clear doorbell data shared with the GuC + * + * These functions modify shared data and so need access to the mapped + * client object which contains the page being used for the doorbell + */ + +static void guc_init_doorbell(struct intel_guc *guc, + struct i915_guc_client *client) +{ + struct guc_doorbell_info *doorbell; + void *base; + + base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0)); + doorbell = base + client->doorbell_offset; + + doorbell->db_status = 1; + doorbell->cookie = 0; + + kunmap_atomic(base); +} + +static int guc_ring_doorbell(struct i915_guc_client *gc) +{ + struct guc_process_desc *desc; + union guc_doorbell_qw db_cmp, db_exc, db_ret; + union guc_doorbell_qw *db; + void *base; + int attempt = 2, ret = -EAGAIN; + + base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, 0)); + desc = base + gc->proc_desc_offset; + + /* Update the tail so it is visible to GuC */ + desc->tail = gc->wq_tail; + + /* current cookie */ + db_cmp.db_status = GUC_DOORBELL_ENABLED; + db_cmp.cookie = gc->cookie; + + /* cookie to be updated */ + db_exc.db_status = GUC_DOORBELL_ENABLED; + db_exc.cookie = gc->cookie + 1; + if (db_exc.cookie == 0) + db_exc.cookie = 1; + + /* pointer of current doorbell cacheline */ + db = base + gc->doorbell_offset; + + while (attempt--) { + /* lets ring the doorbell */ + db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db, + db_cmp.value_qw, db_exc.value_qw); + + /* if the exchange was successfully executed */ + if (db_ret.value_qw == db_cmp.value_qw) { + /* db was successfully rung */ + gc->cookie = db_exc.cookie; + ret = 0; + break; + } + + /* XXX: doorbell was lost and need to acquire it again */ + if (db_ret.db_status == GUC_DOORBELL_DISABLED) + break; + + DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n", + db_cmp.cookie, db_ret.cookie); + + /* update the cookie to newly read cookie from GuC */ + db_cmp.cookie = db_ret.cookie; + db_exc.cookie = db_ret.cookie + 1; + if (db_exc.cookie == 0) + db_exc.cookie = 1; + } + + kunmap_atomic(base); + return ret; +} + +static void guc_disable_doorbell(struct intel_guc *guc, + struct i915_guc_client *client) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct guc_doorbell_info *doorbell; + void *base; + i915_reg_t drbreg = GEN8_DRBREGL(client->doorbell_id); + int value; + + base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0)); + doorbell = base + client->doorbell_offset; + + doorbell->db_status = 0; + + kunmap_atomic(base); + + I915_WRITE(drbreg, I915_READ(drbreg) & ~GEN8_DRB_VALID); + + value = I915_READ(drbreg); + WARN_ON((value & GEN8_DRB_VALID) != 0); + + I915_WRITE(GEN8_DRBREGU(client->doorbell_id), 0); + I915_WRITE(drbreg, 0); + + /* XXX: wait for any interrupts */ + /* XXX: wait for workqueue to drain */ +} + +/* + * Select, assign and relase doorbell cachelines + * + * These functions track which doorbell cachelines are in use. + * The data they manipulate is protected by the host2guc lock. + */ + +static uint32_t select_doorbell_cacheline(struct intel_guc *guc) +{ + const uint32_t cacheline_size = cache_line_size(); + uint32_t offset; + + /* Doorbell uses a single cache line within a page */ + offset = offset_in_page(guc->db_cacheline); + + /* Moving to next cache line to reduce contention */ + guc->db_cacheline += cacheline_size; + + DRM_DEBUG_DRIVER("selected doorbell cacheline 0x%x, next 0x%x, linesize %u\n", + offset, guc->db_cacheline, cacheline_size); + + return offset; +} + +static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority) +{ + /* + * The bitmap is split into two halves; the first half is used for + * normal priority contexts, the second half for high-priority ones. + * Note that logically higher priorities are numerically less than + * normal ones, so the test below means "is it high-priority?" + */ + const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH); + const uint16_t half = GUC_MAX_DOORBELLS / 2; + const uint16_t start = hi_pri ? half : 0; + const uint16_t end = start + half; + uint16_t id; + + id = find_next_zero_bit(guc->doorbell_bitmap, end, start); + if (id == end) + id = GUC_INVALID_DOORBELL_ID; + else + bitmap_set(guc->doorbell_bitmap, id, 1); + + DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n", + hi_pri ? "high" : "normal", id); + + return id; +} + +static void release_doorbell(struct intel_guc *guc, uint16_t id) +{ + bitmap_clear(guc->doorbell_bitmap, id, 1); +} + +/* + * Initialise the process descriptor shared with the GuC firmware. + */ +static void guc_init_proc_desc(struct intel_guc *guc, + struct i915_guc_client *client) +{ + struct guc_process_desc *desc; + void *base; + + base = kmap_atomic(i915_gem_object_get_page(client->client_obj, 0)); + desc = base + client->proc_desc_offset; + + memset(desc, 0, sizeof(*desc)); + + /* + * XXX: pDoorbell and WQVBaseAddress are pointers in process address + * space for ring3 clients (set them as in mmap_ioctl) or kernel + * space for kernel clients (map on demand instead? May make debug + * easier to have it mapped). + */ + desc->wq_base_addr = 0; + desc->db_base_addr = 0; + + desc->context_id = client->ctx_index; + desc->wq_size_bytes = client->wq_size; + desc->wq_status = WQ_STATUS_ACTIVE; + desc->priority = client->priority; + + kunmap_atomic(base); +} + +/* + * Initialise/clear the context descriptor shared with the GuC firmware. + * + * This descriptor tells the GuC where (in GGTT space) to find the important + * data structures relating to this client (doorbell, process descriptor, + * write queue, etc). + */ + +static void guc_init_ctx_desc(struct intel_guc *guc, + struct i915_guc_client *client) +{ + struct intel_context *ctx = client->owner; + struct guc_context_desc desc; + struct sg_table *sg; + int i; + + memset(&desc, 0, sizeof(desc)); + + desc.attribute = GUC_CTX_DESC_ATTR_ACTIVE | GUC_CTX_DESC_ATTR_KERNEL; + desc.context_id = client->ctx_index; + desc.priority = client->priority; + desc.db_id = client->doorbell_id; + + for (i = 0; i < I915_NUM_RINGS; i++) { + struct guc_execlist_context *lrc = &desc.lrc[i]; + struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf; + struct intel_engine_cs *ring; + struct drm_i915_gem_object *obj; + uint64_t ctx_desc; + + /* TODO: We have a design issue to be solved here. Only when we + * receive the first batch, we know which engine is used by the + * user. But here GuC expects the lrc and ring to be pinned. It + * is not an issue for default context, which is the only one + * for now who owns a GuC client. But for future owner of GuC + * client, need to make sure lrc is pinned prior to enter here. + */ + obj = ctx->engine[i].state; + if (!obj) + break; /* XXX: continue? */ + + ring = ringbuf->ring; + ctx_desc = intel_lr_context_descriptor(ctx, ring); + lrc->context_desc = (u32)ctx_desc; + + /* The state page is after PPHWSP */ + lrc->ring_lcra = i915_gem_obj_ggtt_offset(obj) + + LRC_STATE_PN * PAGE_SIZE; + lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) | + (ring->id << GUC_ELC_ENGINE_OFFSET); + + obj = ringbuf->obj; + + lrc->ring_begin = i915_gem_obj_ggtt_offset(obj); + lrc->ring_end = lrc->ring_begin + obj->base.size - 1; + lrc->ring_next_free_location = lrc->ring_begin; + lrc->ring_current_tail_pointer_value = 0; + + desc.engines_used |= (1 << ring->id); + } + + WARN_ON(desc.engines_used == 0); + + /* + * The CPU address is only needed at certain points, so kmap_atomic on + * demand instead of storing it in the ctx descriptor. + * XXX: May make debug easier to have it mapped + */ + desc.db_trigger_cpu = 0; + desc.db_trigger_uk = client->doorbell_offset + + i915_gem_obj_ggtt_offset(client->client_obj); + desc.db_trigger_phy = client->doorbell_offset + + sg_dma_address(client->client_obj->pages->sgl); + + desc.process_desc = client->proc_desc_offset + + i915_gem_obj_ggtt_offset(client->client_obj); + + desc.wq_addr = client->wq_offset + + i915_gem_obj_ggtt_offset(client->client_obj); + + desc.wq_size = client->wq_size; + + /* + * XXX: Take LRCs from an existing intel_context if this is not an + * IsKMDCreatedContext client + */ + desc.desc_private = (uintptr_t)client; + + /* Pool context is pinned already */ + sg = guc->ctx_pool_obj->pages; + sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc), + sizeof(desc) * client->ctx_index); +} + +static void guc_fini_ctx_desc(struct intel_guc *guc, + struct i915_guc_client *client) +{ + struct guc_context_desc desc; + struct sg_table *sg; + + memset(&desc, 0, sizeof(desc)); + + sg = guc->ctx_pool_obj->pages; + sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc), + sizeof(desc) * client->ctx_index); +} + +/* Get valid workqueue item and return it back to offset */ +static int guc_get_workqueue_space(struct i915_guc_client *gc, u32 *offset) +{ + struct guc_process_desc *desc; + void *base; + u32 size = sizeof(struct guc_wq_item); + int ret = -ETIMEDOUT, timeout_counter = 200; + + base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, 0)); + desc = base + gc->proc_desc_offset; + + while (timeout_counter-- > 0) { + if (CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size) >= size) { + *offset = gc->wq_tail; + + /* advance the tail for next workqueue item */ + gc->wq_tail += size; + gc->wq_tail &= gc->wq_size - 1; + + /* this will break the loop */ + timeout_counter = 0; + ret = 0; + } + + if (timeout_counter) + usleep_range(1000, 2000); + }; + + kunmap_atomic(base); + + return ret; +} + +static int guc_add_workqueue_item(struct i915_guc_client *gc, + struct drm_i915_gem_request *rq) +{ + enum intel_ring_id ring_id = rq->ring->id; + struct guc_wq_item *wqi; + void *base; + u32 tail, wq_len, wq_off = 0; + int ret; + + ret = guc_get_workqueue_space(gc, &wq_off); + if (ret) + return ret; + + /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we + * should not have the case where structure wqi is across page, neither + * wrapped to the beginning. This simplifies the implementation below. + * + * XXX: if not the case, we need save data to a temp wqi and copy it to + * workqueue buffer dw by dw. + */ + WARN_ON(sizeof(struct guc_wq_item) != 16); + WARN_ON(wq_off & 3); + + /* wq starts from the page after doorbell / process_desc */ + base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, + (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT)); + wq_off &= PAGE_SIZE - 1; + wqi = (struct guc_wq_item *)((char *)base + wq_off); + + /* len does not include the header */ + wq_len = sizeof(struct guc_wq_item) / sizeof(u32) - 1; + wqi->header = WQ_TYPE_INORDER | + (wq_len << WQ_LEN_SHIFT) | + (ring_id << WQ_TARGET_SHIFT) | + WQ_NO_WCFLUSH_WAIT; + + /* The GuC wants only the low-order word of the context descriptor */ + wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, rq->ring); + + /* The GuC firmware wants the tail index in QWords, not bytes */ + tail = rq->ringbuf->tail >> 3; + wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; + wqi->fence_id = 0; /*XXX: what fence to be here */ + + kunmap_atomic(base); + + return 0; +} + +#define CTX_RING_BUFFER_START 0x08 + +/* Update the ringbuffer pointer in a saved context image */ +static void lr_context_update(struct drm_i915_gem_request *rq) +{ + enum intel_ring_id ring_id = rq->ring->id; + struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring_id].state; + struct drm_i915_gem_object *rb_obj = rq->ringbuf->obj; + struct page *page; + uint32_t *reg_state; + + BUG_ON(!ctx_obj); + WARN_ON(!i915_gem_obj_is_pinned(ctx_obj)); + WARN_ON(!i915_gem_obj_is_pinned(rb_obj)); + + page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); + reg_state = kmap_atomic(page); + + reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj); + + kunmap_atomic(reg_state); +} + +/** + * i915_guc_submit() - Submit commands through GuC + * @client: the guc client where commands will go through + * @rq: request associated with the commands + * + * Return: 0 if succeed + */ +int i915_guc_submit(struct i915_guc_client *client, + struct drm_i915_gem_request *rq) +{ + struct intel_guc *guc = client->guc; + enum intel_ring_id ring_id = rq->ring->id; + int q_ret, b_ret; + + /* Need this because of the deferred pin ctx and ring */ + /* Shall we move this right after ring is pinned? */ + lr_context_update(rq); + + q_ret = guc_add_workqueue_item(client, rq); + if (q_ret == 0) + b_ret = guc_ring_doorbell(client); + + client->submissions[ring_id] += 1; + if (q_ret) { + client->q_fail += 1; + client->retcode = q_ret; + } else if (b_ret) { + client->b_fail += 1; + client->retcode = q_ret = b_ret; + } else { + client->retcode = 0; + } + guc->submissions[ring_id] += 1; + guc->last_seqno[ring_id] = rq->seqno; + + return q_ret; +} + +/* + * Everything below here is concerned with setup & teardown, and is + * therefore not part of the somewhat time-critical batch-submission + * path of i915_guc_submit() above. + */ + +/** + * gem_allocate_guc_obj() - Allocate gem object for GuC usage + * @dev: drm device + * @size: size of object + * + * This is a wrapper to create a gem obj. In order to use it inside GuC, the + * object needs to be pinned lifetime. Also we must pin it to gtt space other + * than [0, GUC_WOPCM_TOP) because this range is reserved inside GuC. + * + * Return: A drm_i915_gem_object if successful, otherwise NULL. + */ +static struct drm_i915_gem_object *gem_allocate_guc_obj(struct drm_device *dev, + u32 size) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_object *obj; + + obj = i915_gem_alloc_object(dev, size); + if (!obj) + return NULL; + + if (i915_gem_object_get_pages(obj)) { + drm_gem_object_unreference(&obj->base); + return NULL; + } + + if (i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, + PIN_OFFSET_BIAS | GUC_WOPCM_TOP)) { + drm_gem_object_unreference(&obj->base); + return NULL; + } + + /* Invalidate GuC TLB to let GuC take the latest updates to GTT. */ + I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); + + return obj; +} + +/** + * gem_release_guc_obj() - Release gem object allocated for GuC usage + * @obj: gem obj to be released + */ +static void gem_release_guc_obj(struct drm_i915_gem_object *obj) +{ + if (!obj) + return; + + if (i915_gem_obj_is_pinned(obj)) + i915_gem_object_ggtt_unpin(obj); + + drm_gem_object_unreference(&obj->base); +} + +static void guc_client_free(struct drm_device *dev, + struct i915_guc_client *client) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + + if (!client) + return; + + if (client->doorbell_id != GUC_INVALID_DOORBELL_ID) { + /* + * First disable the doorbell, then tell the GuC we've + * finished with it, finally deallocate it in our bitmap + */ + guc_disable_doorbell(guc, client); + host2guc_release_doorbell(guc, client); + release_doorbell(guc, client->doorbell_id); + } + + /* + * XXX: wait for any outstanding submissions before freeing memory. + * Be sure to drop any locks + */ + + gem_release_guc_obj(client->client_obj); + + if (client->ctx_index != GUC_INVALID_CTX_ID) { + guc_fini_ctx_desc(guc, client); + ida_simple_remove(&guc->ctx_ids, client->ctx_index); + } + + kfree(client); +} + +/** + * guc_client_alloc() - Allocate an i915_guc_client + * @dev: drm device + * @priority: four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW + * The kernel client to replace ExecList submission is created with + * NORMAL priority. Priority of a client for scheduler can be HIGH, + * while a preemption context can use CRITICAL. + * @ctx: the context that owns the client (we use the default render + * context) + * + * Return: An i915_guc_client object if success. + */ +static struct i915_guc_client *guc_client_alloc(struct drm_device *dev, + uint32_t priority, + struct intel_context *ctx) +{ + struct i915_guc_client *client; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + struct drm_i915_gem_object *obj; + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return NULL; + + client->doorbell_id = GUC_INVALID_DOORBELL_ID; + client->priority = priority; + client->owner = ctx; + client->guc = guc; + + client->ctx_index = (uint32_t)ida_simple_get(&guc->ctx_ids, 0, + GUC_MAX_GPU_CONTEXTS, GFP_KERNEL); + if (client->ctx_index >= GUC_MAX_GPU_CONTEXTS) { + client->ctx_index = GUC_INVALID_CTX_ID; + goto err; + } + + /* The first page is doorbell/proc_desc. Two followed pages are wq. */ + obj = gem_allocate_guc_obj(dev, GUC_DB_SIZE + GUC_WQ_SIZE); + if (!obj) + goto err; + + client->client_obj = obj; + client->wq_offset = GUC_DB_SIZE; + client->wq_size = GUC_WQ_SIZE; + + client->doorbell_offset = select_doorbell_cacheline(guc); + + /* + * Since the doorbell only requires a single cacheline, we can save + * space by putting the application process descriptor in the same + * page. Use the half of the page that doesn't include the doorbell. + */ + if (client->doorbell_offset >= (GUC_DB_SIZE / 2)) + client->proc_desc_offset = 0; + else + client->proc_desc_offset = (GUC_DB_SIZE / 2); + + client->doorbell_id = assign_doorbell(guc, client->priority); + if (client->doorbell_id == GUC_INVALID_DOORBELL_ID) + /* XXX: evict a doorbell instead */ + goto err; + + guc_init_proc_desc(guc, client); + guc_init_ctx_desc(guc, client); + guc_init_doorbell(guc, client); + + /* XXX: Any cache flushes needed? General domain mgmt calls? */ + + if (host2guc_allocate_doorbell(guc, client)) + goto err; + + DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u db_id %u\n", + priority, client, client->ctx_index, client->doorbell_id); + + return client; + +err: + DRM_ERROR("FAILED to create priority %u GuC client!\n", priority); + + guc_client_free(dev, client); + return NULL; +} + +static void guc_create_log(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct drm_i915_gem_object *obj; + unsigned long offset; + uint32_t size, flags; + + if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN) + return; + + if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX) + i915.guc_log_level = GUC_LOG_VERBOSITY_MAX; + + /* The first page is to save log buffer state. Allocate one + * extra page for others in case for overlap */ + size = (1 + GUC_LOG_DPC_PAGES + 1 + + GUC_LOG_ISR_PAGES + 1 + + GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT; + + obj = guc->log_obj; + if (!obj) { + obj = gem_allocate_guc_obj(dev_priv->dev, size); + if (!obj) { + /* logging will be off */ + i915.guc_log_level = -1; + return; + } + + guc->log_obj = obj; + } + + /* each allocated unit is a page */ + flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL | + (GUC_LOG_DPC_PAGES << GUC_LOG_DPC_SHIFT) | + (GUC_LOG_ISR_PAGES << GUC_LOG_ISR_SHIFT) | + (GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT); + + offset = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; /* in pages */ + guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; +} + +/* + * Set up the memory resources to be shared with the GuC. At this point, + * we require just one object that can be mapped through the GGTT. + */ +int i915_guc_submission_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + const size_t ctxsize = sizeof(struct guc_context_desc); + const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize; + const size_t gemsize = round_up(poolsize, PAGE_SIZE); + struct intel_guc *guc = &dev_priv->guc; + + if (!i915.enable_guc_submission) + return 0; /* not enabled */ + + if (guc->ctx_pool_obj) + return 0; /* already allocated */ + + guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv->dev, gemsize); + if (!guc->ctx_pool_obj) + return -ENOMEM; + + ida_init(&guc->ctx_ids); + + guc_create_log(guc); + + return 0; +} + +int i915_guc_submission_enable(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + struct intel_context *ctx = dev_priv->ring[RCS].default_context; + struct i915_guc_client *client; + + /* client for execbuf submission */ + client = guc_client_alloc(dev, GUC_CTX_PRIORITY_KMD_NORMAL, ctx); + if (!client) { + DRM_ERROR("Failed to create execbuf guc_client\n"); + return -ENOMEM; + } + + guc->execbuf_client = client; + + host2guc_sample_forcewake(guc, client); + + return 0; +} + +void i915_guc_submission_disable(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + + guc_client_free(dev, guc->execbuf_client); + guc->execbuf_client = NULL; +} + +void i915_guc_submission_fini(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + + gem_release_guc_obj(dev_priv->guc.log_obj); + guc->log_obj = NULL; + + if (guc->ctx_pool_obj) + ida_destroy(&guc->ctx_ids); + gem_release_guc_obj(guc->ctx_pool_obj); + guc->ctx_pool_obj = NULL; +} + +/** + * intel_guc_suspend() - notify GuC entering suspend state + * @dev: drm device + */ +int intel_guc_suspend(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + struct intel_context *ctx; + u32 data[3]; + + if (!i915.enable_guc_submission) + return 0; + + ctx = dev_priv->ring[RCS].default_context; + + data[0] = HOST2GUC_ACTION_ENTER_S_STATE; + /* any value greater than GUC_POWER_D0 */ + data[1] = GUC_POWER_D1; + /* first page is shared data with GuC */ + data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state); + + return host2guc_action(guc, data, ARRAY_SIZE(data)); +} + + +/** + * intel_guc_resume() - notify GuC resuming from suspend state + * @dev: drm device + */ +int intel_guc_resume(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc *guc = &dev_priv->guc; + struct intel_context *ctx; + u32 data[3]; + + if (!i915.enable_guc_submission) + return 0; + + ctx = dev_priv->ring[RCS].default_context; + + data[0] = HOST2GUC_ACTION_EXIT_S_STATE; + data[1] = GUC_POWER_D0; + /* first page is shared data with GuC */ + data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state); + + return host2guc_action(guc, data, ARRAY_SIZE(data)); +} diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index 23aa04cded6b..97f3a5640289 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -35,107 +35,20 @@ #include <drm/i915_drm.h> #include "i915_drv.h" -typedef struct _drm_i915_batchbuffer32 { - int start; /* agp offset */ - int used; /* nr bytes in use */ - int DR1; /* hw flags for GFX_OP_DRAWRECT_INFO */ - int DR4; /* window origin for GFX_OP_DRAWRECT_INFO */ - int num_cliprects; /* mulitpass with multiple cliprects? */ - u32 cliprects; /* pointer to userspace cliprects */ -} drm_i915_batchbuffer32_t; - -static int compat_i915_batchbuffer(struct file *file, unsigned int cmd, - unsigned long arg) -{ - drm_i915_batchbuffer32_t batchbuffer32; - drm_i915_batchbuffer_t __user *batchbuffer; - - if (copy_from_user - (&batchbuffer32, (void __user *)arg, sizeof(batchbuffer32))) - return -EFAULT; - - batchbuffer = compat_alloc_user_space(sizeof(*batchbuffer)); - if (!access_ok(VERIFY_WRITE, batchbuffer, sizeof(*batchbuffer)) - || __put_user(batchbuffer32.start, &batchbuffer->start) - || __put_user(batchbuffer32.used, &batchbuffer->used) - || __put_user(batchbuffer32.DR1, &batchbuffer->DR1) - || __put_user(batchbuffer32.DR4, &batchbuffer->DR4) - || __put_user(batchbuffer32.num_cliprects, - &batchbuffer->num_cliprects) - || __put_user((int __user *)(unsigned long)batchbuffer32.cliprects, - &batchbuffer->cliprects)) - return -EFAULT; - - return drm_ioctl(file, DRM_IOCTL_I915_BATCHBUFFER, - (unsigned long)batchbuffer); -} - -typedef struct _drm_i915_cmdbuffer32 { - u32 buf; /* pointer to userspace command buffer */ - int sz; /* nr bytes in buf */ - int DR1; /* hw flags for GFX_OP_DRAWRECT_INFO */ - int DR4; /* window origin for GFX_OP_DRAWRECT_INFO */ - int num_cliprects; /* mulitpass with multiple cliprects? */ - u32 cliprects; /* pointer to userspace cliprects */ -} drm_i915_cmdbuffer32_t; - -static int compat_i915_cmdbuffer(struct file *file, unsigned int cmd, - unsigned long arg) -{ - drm_i915_cmdbuffer32_t cmdbuffer32; - drm_i915_cmdbuffer_t __user *cmdbuffer; - - if (copy_from_user - (&cmdbuffer32, (void __user *)arg, sizeof(cmdbuffer32))) - return -EFAULT; - - cmdbuffer = compat_alloc_user_space(sizeof(*cmdbuffer)); - if (!access_ok(VERIFY_WRITE, cmdbuffer, sizeof(*cmdbuffer)) - || __put_user((int __user *)(unsigned long)cmdbuffer32.buf, - &cmdbuffer->buf) - || __put_user(cmdbuffer32.sz, &cmdbuffer->sz) - || __put_user(cmdbuffer32.DR1, &cmdbuffer->DR1) - || __put_user(cmdbuffer32.DR4, &cmdbuffer->DR4) - || __put_user(cmdbuffer32.num_cliprects, &cmdbuffer->num_cliprects) - || __put_user((int __user *)(unsigned long)cmdbuffer32.cliprects, - &cmdbuffer->cliprects)) - return -EFAULT; - - return drm_ioctl(file, DRM_IOCTL_I915_CMDBUFFER, - (unsigned long)cmdbuffer); -} - -typedef struct drm_i915_irq_emit32 { - u32 irq_seq; -} drm_i915_irq_emit32_t; - -static int compat_i915_irq_emit(struct file *file, unsigned int cmd, - unsigned long arg) -{ - drm_i915_irq_emit32_t req32; - drm_i915_irq_emit_t __user *request; - - if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) - return -EFAULT; - - request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) - || __put_user((int __user *)(unsigned long)req32.irq_seq, - &request->irq_seq)) - return -EFAULT; - - return drm_ioctl(file, DRM_IOCTL_I915_IRQ_EMIT, - (unsigned long)request); -} -typedef struct drm_i915_getparam32 { - int param; +struct drm_i915_getparam32 { + s32 param; + /* + * We screwed up the generic ioctl struct here and used a variable-sized + * pointer. Use u32 in the compat struct to match the 32bit pointer + * userspace expects. + */ u32 value; -} drm_i915_getparam32_t; +}; static int compat_i915_getparam(struct file *file, unsigned int cmd, unsigned long arg) { - drm_i915_getparam32_t req32; + struct drm_i915_getparam32 req32; drm_i915_getparam_t __user *request; if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) @@ -152,41 +65,8 @@ static int compat_i915_getparam(struct file *file, unsigned int cmd, (unsigned long)request); } -typedef struct drm_i915_mem_alloc32 { - int region; - int alignment; - int size; - u32 region_offset; /* offset from start of fb or agp */ -} drm_i915_mem_alloc32_t; - -static int compat_i915_alloc(struct file *file, unsigned int cmd, - unsigned long arg) -{ - drm_i915_mem_alloc32_t req32; - drm_i915_mem_alloc_t __user *request; - - if (copy_from_user(&req32, (void __user *)arg, sizeof(req32))) - return -EFAULT; - - request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) - || __put_user(req32.region, &request->region) - || __put_user(req32.alignment, &request->alignment) - || __put_user(req32.size, &request->size) - || __put_user((void __user *)(unsigned long)req32.region_offset, - &request->region_offset)) - return -EFAULT; - - return drm_ioctl(file, DRM_IOCTL_I915_ALLOC, - (unsigned long)request); -} - static drm_ioctl_compat_t *i915_compat_ioctls[] = { - [DRM_I915_BATCHBUFFER] = compat_i915_batchbuffer, - [DRM_I915_CMDBUFFER] = compat_i915_cmdbuffer, [DRM_I915_GETPARAM] = compat_i915_getparam, - [DRM_I915_IRQ_EMIT] = compat_i915_irq_emit, - [DRM_I915_ALLOC] = compat_i915_alloc }; /** diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 984e2fe6688c..e88d692583a5 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -45,6 +45,18 @@ * and related files, but that will be described in separate chapters. */ +static const u32 hpd_ilk[HPD_NUM_PINS] = { + [HPD_PORT_A] = DE_DP_A_HOTPLUG, +}; + +static const u32 hpd_ivb[HPD_NUM_PINS] = { + [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB, +}; + +static const u32 hpd_bdw[HPD_NUM_PINS] = { + [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG, +}; + static const u32 hpd_ibx[HPD_NUM_PINS] = { [HPD_CRT] = SDE_CRT_HOTPLUG, [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG, @@ -61,6 +73,14 @@ static const u32 hpd_cpt[HPD_NUM_PINS] = { [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT }; +static const u32 hpd_spt[HPD_NUM_PINS] = { + [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT, + [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT, + [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT, + [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT, + [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT +}; + static const u32 hpd_mask_i915[HPD_NUM_PINS] = { [HPD_CRT] = CRT_HOTPLUG_INT_EN, [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN, @@ -90,6 +110,7 @@ static const u32 hpd_status_i915[HPD_NUM_PINS] = { /* BXT hpd list */ static const u32 hpd_bxt[HPD_NUM_PINS] = { + [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA, [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB, [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC }; @@ -118,27 +139,31 @@ static const u32 hpd_bxt[HPD_NUM_PINS] = { /* * We should clear IMR at preinstall/uninstall, and just check at postinstall. */ -#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \ - u32 val = I915_READ(reg); \ - if (val) { \ - WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \ - (reg), val); \ - I915_WRITE((reg), 0xffffffff); \ - POSTING_READ(reg); \ - I915_WRITE((reg), 0xffffffff); \ - POSTING_READ(reg); \ - } \ -} while (0) +static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, + i915_reg_t reg) +{ + u32 val = I915_READ(reg); + + if (val == 0) + return; + + WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", + i915_mmio_reg_offset(reg), val); + I915_WRITE(reg, 0xffffffff); + POSTING_READ(reg); + I915_WRITE(reg, 0xffffffff); + POSTING_READ(reg); +} #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \ - GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \ + gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \ I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \ I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \ POSTING_READ(GEN8_##type##_IMR(which)); \ } while (0) #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \ - GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \ + gen5_assert_iir_is_zero(dev_priv, type##IIR); \ I915_WRITE(type##IER, (ier_val)); \ I915_WRITE(type##IMR, (imr_val)); \ POSTING_READ(type##IMR); \ @@ -147,31 +172,68 @@ static const u32 hpd_bxt[HPD_NUM_PINS] = { static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); /* For display hotplug interrupt */ -void -ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask) +static inline void +i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv, + uint32_t mask, + uint32_t bits) { + uint32_t val; + assert_spin_locked(&dev_priv->irq_lock); + WARN_ON(bits & ~mask); - if (WARN_ON(!intel_irqs_enabled(dev_priv))) - return; + val = I915_READ(PORT_HOTPLUG_EN); + val &= ~mask; + val |= bits; + I915_WRITE(PORT_HOTPLUG_EN, val); +} - if ((dev_priv->irq_mask & mask) != 0) { - dev_priv->irq_mask &= ~mask; - I915_WRITE(DEIMR, dev_priv->irq_mask); - POSTING_READ(DEIMR); - } +/** + * i915_hotplug_interrupt_update - update hotplug interrupt enable + * @dev_priv: driver private + * @mask: bits to update + * @bits: bits to enable + * NOTE: the HPD enable bits are modified both inside and outside + * of an interrupt context. To avoid that read-modify-write cycles + * interfer, these bits are protected by a spinlock. Since this + * function is usually not called from a context where the lock is + * held already, this function acquires the lock itself. A non-locking + * version is also available. + */ +void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, + uint32_t mask, + uint32_t bits) +{ + spin_lock_irq(&dev_priv->irq_lock); + i915_hotplug_interrupt_update_locked(dev_priv, mask, bits); + spin_unlock_irq(&dev_priv->irq_lock); } -void -ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask) +/** + * ilk_update_display_irq - update DEIMR + * @dev_priv: driver private + * @interrupt_mask: mask of interrupt bits to update + * @enabled_irq_mask: mask of interrupt bits to enable + */ +void ilk_update_display_irq(struct drm_i915_private *dev_priv, + uint32_t interrupt_mask, + uint32_t enabled_irq_mask) { + uint32_t new_val; + assert_spin_locked(&dev_priv->irq_lock); + WARN_ON(enabled_irq_mask & ~interrupt_mask); + if (WARN_ON(!intel_irqs_enabled(dev_priv))) return; - if ((dev_priv->irq_mask & mask) != mask) { - dev_priv->irq_mask |= mask; + new_val = dev_priv->irq_mask; + new_val &= ~interrupt_mask; + new_val |= (~enabled_irq_mask & interrupt_mask); + + if (new_val != dev_priv->irq_mask) { + dev_priv->irq_mask = new_val; I915_WRITE(DEIMR, dev_priv->irq_mask); POSTING_READ(DEIMR); } @@ -210,27 +272,27 @@ void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) ilk_update_gt_irq(dev_priv, mask, 0); } -static u32 gen6_pm_iir(struct drm_i915_private *dev_priv) +static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv) { return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; } -static u32 gen6_pm_imr(struct drm_i915_private *dev_priv) +static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv) { return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR; } -static u32 gen6_pm_ier(struct drm_i915_private *dev_priv) +static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv) { return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER; } /** - * snb_update_pm_irq - update GEN6_PMIMR - * @dev_priv: driver private - * @interrupt_mask: mask of interrupt bits to update - * @enabled_irq_mask: mask of interrupt bits to enable - */ + * snb_update_pm_irq - update GEN6_PMIMR + * @dev_priv: driver private + * @interrupt_mask: mask of interrupt bits to update + * @enabled_irq_mask: mask of interrupt bits to enable + */ static void snb_update_pm_irq(struct drm_i915_private *dev_priv, uint32_t interrupt_mask, uint32_t enabled_irq_mask) @@ -277,7 +339,7 @@ void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) void gen6_reset_rps_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t reg = gen6_pm_iir(dev_priv); + i915_reg_t reg = gen6_pm_iir(dev_priv); spin_lock_irq(&dev_priv->irq_lock); I915_WRITE(reg, dev_priv->pm_rps_events); @@ -344,6 +406,70 @@ void gen6_disable_rps_interrupts(struct drm_device *dev) } /** + * bdw_update_port_irq - update DE port interrupt + * @dev_priv: driver private + * @interrupt_mask: mask of interrupt bits to update + * @enabled_irq_mask: mask of interrupt bits to enable + */ +static void bdw_update_port_irq(struct drm_i915_private *dev_priv, + uint32_t interrupt_mask, + uint32_t enabled_irq_mask) +{ + uint32_t new_val; + uint32_t old_val; + + assert_spin_locked(&dev_priv->irq_lock); + + WARN_ON(enabled_irq_mask & ~interrupt_mask); + + if (WARN_ON(!intel_irqs_enabled(dev_priv))) + return; + + old_val = I915_READ(GEN8_DE_PORT_IMR); + + new_val = old_val; + new_val &= ~interrupt_mask; + new_val |= (~enabled_irq_mask & interrupt_mask); + + if (new_val != old_val) { + I915_WRITE(GEN8_DE_PORT_IMR, new_val); + POSTING_READ(GEN8_DE_PORT_IMR); + } +} + +/** + * bdw_update_pipe_irq - update DE pipe interrupt + * @dev_priv: driver private + * @pipe: pipe whose interrupt to update + * @interrupt_mask: mask of interrupt bits to update + * @enabled_irq_mask: mask of interrupt bits to enable + */ +void bdw_update_pipe_irq(struct drm_i915_private *dev_priv, + enum pipe pipe, + uint32_t interrupt_mask, + uint32_t enabled_irq_mask) +{ + uint32_t new_val; + + assert_spin_locked(&dev_priv->irq_lock); + + WARN_ON(enabled_irq_mask & ~interrupt_mask); + + if (WARN_ON(!intel_irqs_enabled(dev_priv))) + return; + + new_val = dev_priv->de_irq_mask[pipe]; + new_val &= ~interrupt_mask; + new_val |= (~enabled_irq_mask & interrupt_mask); + + if (new_val != dev_priv->de_irq_mask[pipe]) { + dev_priv->de_irq_mask[pipe] = new_val; + I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); + POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); + } +} + +/** * ibx_display_interrupt_update - update SDEIMR * @dev_priv: driver private * @interrupt_mask: mask of interrupt bits to update @@ -372,7 +498,7 @@ static void __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, u32 enable_mask, u32 status_mask) { - u32 reg = PIPESTAT(pipe); + i915_reg_t reg = PIPESTAT(pipe); u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; assert_spin_locked(&dev_priv->irq_lock); @@ -399,7 +525,7 @@ static void __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, u32 enable_mask, u32 status_mask) { - u32 reg = PIPESTAT(pipe); + i915_reg_t reg = PIPESTAT(pipe); u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; assert_spin_locked(&dev_priv->irq_lock); @@ -479,6 +605,7 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, /** * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion + * @dev: drm device */ static void i915_enable_asle_pipestat(struct drm_device *dev) { @@ -547,7 +674,7 @@ static void i915_enable_asle_pipestat(struct drm_device *dev) * of horizontal active on the first line of vertical active */ -static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe) +static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe) { /* Gen2 doesn't have a hardware frame counter */ return 0; @@ -556,16 +683,14 @@ static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe) /* Called from drm generic code, passed a 'crtc', which * we use as a pipe index */ -static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) +static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long high_frame; - unsigned long low_frame; + i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; struct intel_crtc *intel_crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); - const struct drm_display_mode *mode = - &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *mode = &intel_crtc->base.hwmode; htotal = mode->crtc_htotal; hsync_start = mode->crtc_hsync_start; @@ -605,22 +730,19 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff; } -static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe) +static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; - int reg = PIPE_FRMCOUNT_GM45(pipe); - return I915_READ(reg); + return I915_READ(PIPE_FRMCOUNT_G4X(pipe)); } -/* raw reads, only for fast reads of display block, no need for forcewake etc. */ -#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__)) - +/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *mode = &crtc->base.hwmode; enum pipe pipe = crtc->pipe; int position, vtotal; @@ -629,9 +751,35 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) vtotal /= 2; if (IS_GEN2(dev)) - position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; + position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; else - position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; + position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; + + /* + * On HSW, the DSL reg (0x70000) appears to return 0 if we + * read it just before the start of vblank. So try it again + * so we don't accidentally end up spanning a vblank frame + * increment, causing the pipe_update_end() code to squak at us. + * + * The nature of this problem means we can't simply check the ISR + * bit and return the vblank start value; nor can we use the scanline + * debug register in the transcoder as it appears to have the same + * problem. We may need to extend this to include other platforms, + * but so far testing only shows the problem on HSW. + */ + if (HAS_DDI(dev) && !position) { + int i, temp; + + for (i = 0; i < 100; i++) { + udelay(1); + temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & + DSL_LINEMASK_GEN3; + if (temp != position) { + position = temp; + break; + } + } + } /* * See update_scanline_offset() for the details on the @@ -640,21 +788,21 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) return (position + crtc->scanline_offset) % vtotal; } -static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, +static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, unsigned int flags, int *vpos, int *hpos, - ktime_t *stime, ktime_t *etime) + ktime_t *stime, ktime_t *etime, + const struct drm_display_mode *mode) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - const struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode; int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; bool in_vbl = true; int ret = 0; unsigned long irqflags; - if (!intel_crtc->active) { + if (WARN_ON(!mode->crtc_clock)) { DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled " "pipe %c\n", pipe_name(pipe)); return 0; @@ -697,7 +845,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, * We can split this into vertical and horizontal * scanout position. */ - position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; + position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; /* convert to pixel counts */ vbl_start *= htotal; @@ -777,179 +925,34 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc) return position; } -static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe, +static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, int *max_error, struct timeval *vblank_time, unsigned flags) { struct drm_crtc *crtc; - if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) { - DRM_ERROR("Invalid crtc %d\n", pipe); + if (pipe >= INTEL_INFO(dev)->num_pipes) { + DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } /* Get drm_crtc to timestamp: */ crtc = intel_get_crtc_for_pipe(dev, pipe); if (crtc == NULL) { - DRM_ERROR("Invalid crtc %d\n", pipe); + DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } - if (!crtc->state->enable) { - DRM_DEBUG_KMS("crtc %d is disabled\n", pipe); + if (!crtc->hwmode.crtc_clock) { + DRM_DEBUG_KMS("crtc %u is disabled\n", pipe); return -EBUSY; } /* Helper routine in DRM core does all the work: */ return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, vblank_time, flags, - crtc, - &to_intel_crtc(crtc)->config->base.adjusted_mode); -} - -static bool intel_hpd_irq_event(struct drm_device *dev, - struct drm_connector *connector) -{ - enum drm_connector_status old_status; - - WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); - old_status = connector->status; - - connector->status = connector->funcs->detect(connector, false); - if (old_status == connector->status) - return false; - - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n", - connector->base.id, - connector->name, - drm_get_connector_status_name(old_status), - drm_get_connector_status_name(connector->status)); - - return true; -} - -static void i915_digport_work_func(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, struct drm_i915_private, dig_port_work); - u32 long_port_mask, short_port_mask; - struct intel_digital_port *intel_dig_port; - int i; - u32 old_bits = 0; - - spin_lock_irq(&dev_priv->irq_lock); - long_port_mask = dev_priv->long_hpd_port_mask; - dev_priv->long_hpd_port_mask = 0; - short_port_mask = dev_priv->short_hpd_port_mask; - dev_priv->short_hpd_port_mask = 0; - spin_unlock_irq(&dev_priv->irq_lock); - - for (i = 0; i < I915_MAX_PORTS; i++) { - bool valid = false; - bool long_hpd = false; - intel_dig_port = dev_priv->hpd_irq_port[i]; - if (!intel_dig_port || !intel_dig_port->hpd_pulse) - continue; - - if (long_port_mask & (1 << i)) { - valid = true; - long_hpd = true; - } else if (short_port_mask & (1 << i)) - valid = true; - - if (valid) { - enum irqreturn ret; - - ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd); - if (ret == IRQ_NONE) { - /* fall back to old school hpd */ - old_bits |= (1 << intel_dig_port->base.hpd_pin); - } - } - } - - if (old_bits) { - spin_lock_irq(&dev_priv->irq_lock); - dev_priv->hpd_event_bits |= old_bits; - spin_unlock_irq(&dev_priv->irq_lock); - schedule_work(&dev_priv->hotplug_work); - } -} - -/* - * Handle hotplug events outside the interrupt handler proper. - */ -#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000) - -static void i915_hotplug_work_func(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, struct drm_i915_private, hotplug_work); - struct drm_device *dev = dev_priv->dev; - struct drm_mode_config *mode_config = &dev->mode_config; - struct intel_connector *intel_connector; - struct intel_encoder *intel_encoder; - struct drm_connector *connector; - bool hpd_disabled = false; - bool changed = false; - u32 hpd_event_bits; - - mutex_lock(&mode_config->mutex); - DRM_DEBUG_KMS("running encoder hotplug functions\n"); - - spin_lock_irq(&dev_priv->irq_lock); - - hpd_event_bits = dev_priv->hpd_event_bits; - dev_priv->hpd_event_bits = 0; - list_for_each_entry(connector, &mode_config->connector_list, head) { - intel_connector = to_intel_connector(connector); - if (!intel_connector->encoder) - continue; - intel_encoder = intel_connector->encoder; - if (intel_encoder->hpd_pin > HPD_NONE && - dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED && - connector->polled == DRM_CONNECTOR_POLL_HPD) { - DRM_INFO("HPD interrupt storm detected on connector %s: " - "switching from hotplug detection to polling\n", - connector->name); - dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED; - connector->polled = DRM_CONNECTOR_POLL_CONNECT - | DRM_CONNECTOR_POLL_DISCONNECT; - hpd_disabled = true; - } - if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) { - DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n", - connector->name, intel_encoder->hpd_pin); - } - } - /* if there were no outputs to poll, poll was disabled, - * therefore make sure it's enabled when disabling HPD on - * some connectors */ - if (hpd_disabled) { - drm_kms_helper_poll_enable(dev); - mod_delayed_work(system_wq, &dev_priv->hotplug_reenable_work, - msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY)); - } - - spin_unlock_irq(&dev_priv->irq_lock); - - list_for_each_entry(connector, &mode_config->connector_list, head) { - intel_connector = to_intel_connector(connector); - if (!intel_connector->encoder) - continue; - intel_encoder = intel_connector->encoder; - if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) { - if (intel_encoder->hot_plug) - intel_encoder->hot_plug(intel_encoder); - if (intel_hpd_irq_event(dev, connector)) - changed = true; - } - } - mutex_unlock(&mode_config->mutex); - - if (changed) - drm_kms_helper_hotplug_event(dev); + &crtc->hwmode); } static void ironlake_rps_change_irq_handler(struct drm_device *dev) @@ -1015,12 +1018,16 @@ static bool vlv_c0_above(struct drm_i915_private *dev_priv, int threshold) { u64 time, c0; + unsigned int mul = 100; if (old->cz_clock == 0) return false; + if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH) + mul <<= 8; + time = now->cz_clock - old->cz_clock; - time *= threshold * dev_priv->mem_freq; + time *= threshold * dev_priv->czclk_freq; /* Workload can be split between render + media, e.g. SwapBuffers * being blitted in X after being rendered in mesa. To account for @@ -1028,7 +1035,7 @@ static bool vlv_c0_above(struct drm_i915_private *dev_priv, */ c0 = now->render_c0 - old->render_c0; c0 += now->media_c0 - old->media_c0; - c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000; + c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC; return c0 >= time; } @@ -1199,7 +1206,7 @@ static void ivybridge_parity_work(struct work_struct *work) POSTING_READ(GEN7_MISCCPCTL); while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) { - u32 reg; + i915_reg_t reg; slice--; if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev))) @@ -1207,7 +1214,7 @@ static void ivybridge_parity_work(struct work_struct *work) dev_priv->l3_parity.which_slice &= ~(1<<slice); - reg = GEN7_L3CDERRST1 + (slice * 0x200); + reg = GEN7_L3CDERRST1(slice); error_status = I915_READ(reg); row = GEN7_PARITY_ERROR_ROW(error_status); @@ -1301,70 +1308,69 @@ static void snb_gt_irq_handler(struct drm_device *dev, ivybridge_parity_error_irq_handler(dev, gt_iir); } +static __always_inline void +gen8_cs_irq_handler(struct intel_engine_cs *ring, u32 iir, int test_shift) +{ + if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) + notify_ring(ring); + if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) + intel_lrc_irq_handler(ring); +} + static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) { irqreturn_t ret = IRQ_NONE; if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) { - u32 tmp = I915_READ_FW(GEN8_GT_IIR(0)); - if (tmp) { - I915_WRITE_FW(GEN8_GT_IIR(0), tmp); + u32 iir = I915_READ_FW(GEN8_GT_IIR(0)); + if (iir) { + I915_WRITE_FW(GEN8_GT_IIR(0), iir); ret = IRQ_HANDLED; - if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT)) - intel_lrc_irq_handler(&dev_priv->ring[RCS]); - if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT)) - notify_ring(&dev_priv->ring[RCS]); + gen8_cs_irq_handler(&dev_priv->ring[RCS], + iir, GEN8_RCS_IRQ_SHIFT); - if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT)) - intel_lrc_irq_handler(&dev_priv->ring[BCS]); - if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT)) - notify_ring(&dev_priv->ring[BCS]); + gen8_cs_irq_handler(&dev_priv->ring[BCS], + iir, GEN8_BCS_IRQ_SHIFT); } else DRM_ERROR("The master control interrupt lied (GT0)!\n"); } if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) { - u32 tmp = I915_READ_FW(GEN8_GT_IIR(1)); - if (tmp) { - I915_WRITE_FW(GEN8_GT_IIR(1), tmp); + u32 iir = I915_READ_FW(GEN8_GT_IIR(1)); + if (iir) { + I915_WRITE_FW(GEN8_GT_IIR(1), iir); ret = IRQ_HANDLED; - if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT)) - intel_lrc_irq_handler(&dev_priv->ring[VCS]); - if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT)) - notify_ring(&dev_priv->ring[VCS]); + gen8_cs_irq_handler(&dev_priv->ring[VCS], + iir, GEN8_VCS1_IRQ_SHIFT); - if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT)) - intel_lrc_irq_handler(&dev_priv->ring[VCS2]); - if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT)) - notify_ring(&dev_priv->ring[VCS2]); + gen8_cs_irq_handler(&dev_priv->ring[VCS2], + iir, GEN8_VCS2_IRQ_SHIFT); } else DRM_ERROR("The master control interrupt lied (GT1)!\n"); } if (master_ctl & GEN8_GT_VECS_IRQ) { - u32 tmp = I915_READ_FW(GEN8_GT_IIR(3)); - if (tmp) { - I915_WRITE_FW(GEN8_GT_IIR(3), tmp); + u32 iir = I915_READ_FW(GEN8_GT_IIR(3)); + if (iir) { + I915_WRITE_FW(GEN8_GT_IIR(3), iir); ret = IRQ_HANDLED; - if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT)) - intel_lrc_irq_handler(&dev_priv->ring[VECS]); - if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT)) - notify_ring(&dev_priv->ring[VECS]); + gen8_cs_irq_handler(&dev_priv->ring[VECS], + iir, GEN8_VECS_IRQ_SHIFT); } else DRM_ERROR("The master control interrupt lied (GT3)!\n"); } if (master_ctl & GEN8_GT_PM_IRQ) { - u32 tmp = I915_READ_FW(GEN8_GT_IIR(2)); - if (tmp & dev_priv->pm_rps_events) { + u32 iir = I915_READ_FW(GEN8_GT_IIR(2)); + if (iir & dev_priv->pm_rps_events) { I915_WRITE_FW(GEN8_GT_IIR(2), - tmp & dev_priv->pm_rps_events); + iir & dev_priv->pm_rps_events); ret = IRQ_HANDLED; - gen6_rps_irq_handler(dev_priv, tmp); + gen6_rps_irq_handler(dev_priv, iir); } else DRM_ERROR("The master control interrupt lied (PM)!\n"); } @@ -1372,165 +1378,115 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv, return ret; } -#define HPD_STORM_DETECT_PERIOD 1000 -#define HPD_STORM_THRESHOLD 5 - -static int pch_port_to_hotplug_shift(enum port port) +static bool bxt_port_hotplug_long_detect(enum port port, u32 val) { switch (port) { case PORT_A: + return val & PORTA_HOTPLUG_LONG_DETECT; + case PORT_B: + return val & PORTB_HOTPLUG_LONG_DETECT; + case PORT_C: + return val & PORTC_HOTPLUG_LONG_DETECT; + default: + return false; + } +} + +static bool spt_port_hotplug2_long_detect(enum port port, u32 val) +{ + switch (port) { case PORT_E: + return val & PORTE_HOTPLUG_LONG_DETECT; default: - return -1; + return false; + } +} + +static bool spt_port_hotplug_long_detect(enum port port, u32 val) +{ + switch (port) { + case PORT_A: + return val & PORTA_HOTPLUG_LONG_DETECT; case PORT_B: - return 0; + return val & PORTB_HOTPLUG_LONG_DETECT; case PORT_C: - return 8; + return val & PORTC_HOTPLUG_LONG_DETECT; case PORT_D: - return 16; + return val & PORTD_HOTPLUG_LONG_DETECT; + default: + return false; } } -static int i915_port_to_hotplug_shift(enum port port) +static bool ilk_port_hotplug_long_detect(enum port port, u32 val) { switch (port) { case PORT_A: - case PORT_E: + return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT; default: - return -1; + return false; + } +} + +static bool pch_port_hotplug_long_detect(enum port port, u32 val) +{ + switch (port) { case PORT_B: - return 17; + return val & PORTB_HOTPLUG_LONG_DETECT; case PORT_C: - return 19; + return val & PORTC_HOTPLUG_LONG_DETECT; case PORT_D: - return 21; + return val & PORTD_HOTPLUG_LONG_DETECT; + default: + return false; } } -static enum port get_port_from_pin(enum hpd_pin pin) +static bool i9xx_port_hotplug_long_detect(enum port port, u32 val) { - switch (pin) { - case HPD_PORT_B: - return PORT_B; - case HPD_PORT_C: - return PORT_C; - case HPD_PORT_D: - return PORT_D; + switch (port) { + case PORT_B: + return val & PORTB_HOTPLUG_INT_LONG_PULSE; + case PORT_C: + return val & PORTC_HOTPLUG_INT_LONG_PULSE; + case PORT_D: + return val & PORTD_HOTPLUG_INT_LONG_PULSE; default: - return PORT_A; /* no hpd */ + return false; } } -static void intel_hpd_irq_handler(struct drm_device *dev, - u32 hotplug_trigger, - u32 dig_hotplug_reg, - const u32 hpd[HPD_NUM_PINS]) +/* + * Get a bit mask of pins that have triggered, and which ones may be long. + * This can be called multiple times with the same masks to accumulate + * hotplug detection results from several registers. + * + * Note that the caller is expected to zero out the masks initially. + */ +static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask, + u32 hotplug_trigger, u32 dig_hotplug_reg, + const u32 hpd[HPD_NUM_PINS], + bool long_pulse_detect(enum port port, u32 val)) { - struct drm_i915_private *dev_priv = dev->dev_private; - int i; enum port port; - bool storm_detected = false; - bool queue_dig = false, queue_hp = false; - u32 dig_shift; - u32 dig_port_mask = 0; - - if (!hotplug_trigger) - return; - - DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x\n", - hotplug_trigger, dig_hotplug_reg); + int i; - spin_lock(&dev_priv->irq_lock); - for (i = 1; i < HPD_NUM_PINS; i++) { - if (!(hpd[i] & hotplug_trigger)) + for_each_hpd_pin(i) { + if ((hpd[i] & hotplug_trigger) == 0) continue; - port = get_port_from_pin(i); - if (port && dev_priv->hpd_irq_port[port]) { - bool long_hpd; - - if (!HAS_GMCH_DISPLAY(dev_priv)) { - dig_shift = pch_port_to_hotplug_shift(port); - long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT; - } else { - dig_shift = i915_port_to_hotplug_shift(port); - long_hpd = (hotplug_trigger >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT; - } - - DRM_DEBUG_DRIVER("digital hpd port %c - %s\n", - port_name(port), - long_hpd ? "long" : "short"); - /* for long HPD pulses we want to have the digital queue happen, - but we still want HPD storm detection to function. */ - if (long_hpd) { - dev_priv->long_hpd_port_mask |= (1 << port); - dig_port_mask |= hpd[i]; - } else { - /* for short HPD just trigger the digital queue */ - dev_priv->short_hpd_port_mask |= (1 << port); - hotplug_trigger &= ~hpd[i]; - } - queue_dig = true; - } - } - - for (i = 1; i < HPD_NUM_PINS; i++) { - if (hpd[i] & hotplug_trigger && - dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) { - /* - * On GMCH platforms the interrupt mask bits only - * prevent irq generation, not the setting of the - * hotplug bits itself. So only WARN about unexpected - * interrupts on saner platforms. - */ - WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev), - "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n", - hotplug_trigger, i, hpd[i]); - - continue; - } + *pin_mask |= BIT(i); - if (!(hpd[i] & hotplug_trigger) || - dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED) + if (!intel_hpd_pin_to_port(i, &port)) continue; - if (!(dig_port_mask & hpd[i])) { - dev_priv->hpd_event_bits |= (1 << i); - queue_hp = true; - } - - if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies, - dev_priv->hpd_stats[i].hpd_last_jiffies - + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) { - dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies; - dev_priv->hpd_stats[i].hpd_cnt = 0; - DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i); - } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) { - dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED; - dev_priv->hpd_event_bits &= ~(1 << i); - DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i); - storm_detected = true; - } else { - dev_priv->hpd_stats[i].hpd_cnt++; - DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i, - dev_priv->hpd_stats[i].hpd_cnt); - } + if (long_pulse_detect(port, dig_hotplug_reg)) + *long_mask |= BIT(i); } - if (storm_detected) - dev_priv->display.hpd_irq_setup(dev); - spin_unlock(&dev_priv->irq_lock); + DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n", + hotplug_trigger, dig_hotplug_reg, *pin_mask); - /* - * Our hotplug handler can grab modeset locks (by calling down into the - * fb helpers). Hence it must not be run on our own dev-priv->wq work - * queue for otherwise the flush_work in the pageflip code will - * deadlock. - */ - if (queue_dig) - queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work); - if (queue_hp) - schedule_work(&dev_priv->hotplug_work); } static void gmbus_irq_handler(struct drm_device *dev) @@ -1686,7 +1642,7 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir) spin_lock(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - int reg; + i915_reg_t reg; u32 mask, iir_bit = 0; /* @@ -1755,28 +1711,40 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); + u32 pin_mask = 0, long_mask = 0; - if (hotplug_status) { - I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); - /* - * Make sure hotplug status is cleared before we clear IIR, or else we - * may miss hotplug events. - */ - POSTING_READ(PORT_HOTPLUG_STAT); + if (!hotplug_status) + return; - if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) { - u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; + I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); + /* + * Make sure hotplug status is cleared before we clear IIR, or else we + * may miss hotplug events. + */ + POSTING_READ(PORT_HOTPLUG_STAT); - intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_g4x); - } else { - u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; + if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) { + u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; + + if (hotplug_trigger) { + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, + hotplug_trigger, hpd_status_g4x, + i9xx_port_hotplug_long_detect); - intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_i915); + intel_hpd_irq_handler(dev, pin_mask, long_mask); } - if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) && - hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) + if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) dp_aux_irq_handler(dev); + } else { + u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; + + if (hotplug_trigger) { + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, + hotplug_trigger, hpd_status_i915, + i9xx_port_hotplug_long_detect); + intel_hpd_irq_handler(dev, pin_mask, long_mask); + } } } @@ -1870,17 +1838,45 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg) return ret; } +static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, + const u32 hpd[HPD_NUM_PINS]) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; + + /* + * Somehow the PCH doesn't seem to really ack the interrupt to the CPU + * unless we touch the hotplug register, even if hotplug_trigger is + * zero. Not acking leads to "The master control interrupt lied (SDE)!" + * errors. + */ + dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); + if (!hotplug_trigger) { + u32 mask = PORTA_HOTPLUG_STATUS_MASK | + PORTD_HOTPLUG_STATUS_MASK | + PORTC_HOTPLUG_STATUS_MASK | + PORTB_HOTPLUG_STATUS_MASK; + dig_hotplug_reg &= ~mask; + } + + I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); + if (!hotplug_trigger) + return; + + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, + dig_hotplug_reg, hpd, + pch_port_hotplug_long_detect); + + intel_hpd_irq_handler(dev, pin_mask, long_mask); +} + static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) { struct drm_i915_private *dev_priv = dev->dev_private; int pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; - u32 dig_hotplug_reg; - - dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); - I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); - intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_ibx); + ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx); if (pch_iir & SDE_AUDIO_POWER_MASK) { int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >> @@ -1972,12 +1968,8 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; - u32 dig_hotplug_reg; - dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); - I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); - - intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_cpt); + ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt); if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) { int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >> @@ -2008,10 +2000,67 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) cpt_serr_int_handler(dev); } +static void spt_irq_handler(struct drm_device *dev, u32 pch_iir) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT & + ~SDE_PORTE_HOTPLUG_SPT; + u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT; + u32 pin_mask = 0, long_mask = 0; + + if (hotplug_trigger) { + u32 dig_hotplug_reg; + + dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); + I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); + + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, + dig_hotplug_reg, hpd_spt, + spt_port_hotplug_long_detect); + } + + if (hotplug2_trigger) { + u32 dig_hotplug_reg; + + dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2); + I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg); + + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger, + dig_hotplug_reg, hpd_spt, + spt_port_hotplug2_long_detect); + } + + if (pin_mask) + intel_hpd_irq_handler(dev, pin_mask, long_mask); + + if (pch_iir & SDE_GMBUS_CPT) + gmbus_irq_handler(dev); +} + +static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, + const u32 hpd[HPD_NUM_PINS]) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; + + dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); + I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg); + + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, + dig_hotplug_reg, hpd, + ilk_port_hotplug_long_detect); + + intel_hpd_irq_handler(dev, pin_mask, long_mask); +} + static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir) { struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; + u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG; + + if (hotplug_trigger) + ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk); if (de_iir & DE_AUX_CHANNEL_A) dp_aux_irq_handler(dev); @@ -2061,6 +2110,10 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) { struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; + u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB; + + if (hotplug_trigger) + ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb); if (de_iir & DE_ERR_INT_IVB) ivb_err_int_handler(dev); @@ -2173,36 +2226,20 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) return ret; } -static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status) +static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, + const u32 hpd[HPD_NUM_PINS]) { - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t hp_control; - uint32_t hp_trigger; - - /* Get the status */ - hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK; - hp_control = I915_READ(BXT_HOTPLUG_CTL); - - /* Hotplug not enabled ? */ - if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) { - DRM_ERROR("Interrupt when HPD disabled\n"); - return; - } - - DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", - hp_control & BXT_HOTPLUG_CTL_MASK); + struct drm_i915_private *dev_priv = to_i915(dev); + u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; - /* Check for HPD storm and schedule bottom half */ - intel_hpd_irq_handler(dev, hp_trigger, hp_control, hpd_bxt); + dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); + I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); - /* - * FIXME: Save the hot plug status for bottom half before - * clearing the sticky status bits, else the status will be - * lost. - */ + intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, + dig_hotplug_reg, hpd, + bxt_port_hotplug_long_detect); - /* Clear sticky bits in hpd status */ - I915_WRITE(BXT_HOTPLUG_CTL, hp_control); + intel_hpd_irq_handler(dev, pin_mask, long_mask); } static irqreturn_t gen8_irq_handler(int irq, void *arg) @@ -2218,7 +2255,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) if (!intel_irqs_enabled(dev_priv)) return IRQ_NONE; - if (IS_GEN9(dev)) + if (INTEL_INFO(dev_priv)->gen >= 9) aux_mask |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | GEN9_AUX_CHANNEL_D; @@ -2251,6 +2288,12 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) tmp = I915_READ(GEN8_DE_PORT_IIR); if (tmp) { bool found = false; + u32 hotplug_trigger = 0; + + if (IS_BROXTON(dev_priv)) + hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK; + else if (IS_BROADWELL(dev_priv)) + hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG; I915_WRITE(GEN8_DE_PORT_IIR, tmp); ret = IRQ_HANDLED; @@ -2260,8 +2303,11 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) found = true; } - if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) { - bxt_hpd_handler(dev, tmp); + if (hotplug_trigger) { + if (IS_BROXTON(dev)) + bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt); + else + ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw); found = true; } @@ -2292,7 +2338,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) intel_pipe_handle_vblank(dev, pipe)) intel_check_page_flip(dev, pipe); - if (IS_GEN9(dev)) + if (INTEL_INFO(dev_priv)->gen >= 9) flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE; else flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE; @@ -2310,7 +2356,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) pipe); - if (IS_GEN9(dev)) + if (INTEL_INFO(dev_priv)->gen >= 9) fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS; else fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS; @@ -2334,7 +2380,11 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) if (pch_iir) { I915_WRITE(SDEIIR, pch_iir); ret = IRQ_HANDLED; - cpt_irq_handler(dev, pch_iir); + + if (HAS_PCH_SPT(dev_priv)) + spt_irq_handler(dev, pch_iir); + else + cpt_irq_handler(dev, pch_iir); } else DRM_ERROR("The master control interrupt lied (SDE)!\n"); @@ -2376,6 +2426,7 @@ static void i915_error_wake_up(struct drm_i915_private *dev_priv, /** * i915_reset_and_wakeup - do process context error handling work + * @dev: drm device * * Fire an error uevent so userspace can see that a hang or error * was detected. @@ -2446,7 +2497,7 @@ static void i915_reset_and_wakeup(struct drm_device *dev) kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, reset_done_event); } else { - atomic_set_mask(I915_WEDGED, &error->reset_counter); + atomic_or(I915_WEDGED, &error->reset_counter); } /* @@ -2553,7 +2604,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) * i915_handle_error - handle a gpu error * @dev: drm device * - * Do some basic checking of regsiter state at error time and + * Do some basic checking of register state at error time and * dump it to the syslog. Also call i915_capture_error_state() to make * sure we get a record and make it available in debugfs. Fire a uevent * so userspace knows something bad happened (should trigger collection @@ -2574,7 +2625,7 @@ void i915_handle_error(struct drm_device *dev, bool wedged, i915_report_and_clear_eir(dev); if (wedged) { - atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG, + atomic_or(I915_RESET_IN_PROGRESS_FLAG, &dev_priv->gpu_error.reset_counter); /* @@ -2599,7 +2650,7 @@ void i915_handle_error(struct drm_device *dev, bool wedged, /* Called from drm generic code, passed 'crtc' which * we use as a pipe index */ -static int i915_enable_vblank(struct drm_device *dev, int pipe) +static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; @@ -2616,7 +2667,7 @@ static int i915_enable_vblank(struct drm_device *dev, int pipe) return 0; } -static int ironlake_enable_vblank(struct drm_device *dev, int pipe) +static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; @@ -2624,13 +2675,13 @@ static int ironlake_enable_vblank(struct drm_device *dev, int pipe) DE_PIPE_VBLANK(pipe); spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - ironlake_enable_display_irq(dev_priv, bit); + ilk_enable_display_irq(dev_priv, bit); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); return 0; } -static int valleyview_enable_vblank(struct drm_device *dev, int pipe) +static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; @@ -2643,23 +2694,22 @@ static int valleyview_enable_vblank(struct drm_device *dev, int pipe) return 0; } -static int gen8_enable_vblank(struct drm_device *dev, int pipe) +static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK; - I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); - POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); + bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + return 0; } /* Called from drm generic code, passed 'crtc' which * we use as a pipe index */ -static void i915_disable_vblank(struct drm_device *dev, int pipe) +static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; @@ -2671,7 +2721,7 @@ static void i915_disable_vblank(struct drm_device *dev, int pipe) spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -static void ironlake_disable_vblank(struct drm_device *dev, int pipe) +static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; @@ -2679,11 +2729,11 @@ static void ironlake_disable_vblank(struct drm_device *dev, int pipe) DE_PIPE_VBLANK(pipe); spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - ironlake_disable_display_irq(dev_priv, bit); + ilk_disable_display_irq(dev_priv, bit); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -static void valleyview_disable_vblank(struct drm_device *dev, int pipe) +static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; @@ -2694,15 +2744,13 @@ static void valleyview_disable_vblank(struct drm_device *dev, int pipe) spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -static void gen8_disable_vblank(struct drm_device *dev, int pipe) +static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK; - I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); - POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); + bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } @@ -2766,6 +2814,26 @@ semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno) u64 offset = 0; int i, backwards; + /* + * This function does not support execlist mode - any attempt to + * proceed further into this function will result in a kernel panic + * when dereferencing ring->buffer, which is not set up in execlist + * mode. + * + * The correct way of doing it would be to derive the currently + * executing ring buffer from the current context, which is derived + * from the currently running request. Unfortunately, to get the + * current request we would have to grab the struct_mutex before doing + * anything else, which would be ill-advised since some other thread + * might have grabbed it already and managed to hang itself, causing + * the hang checker to deadlock. + * + * Therefore, this function does not support execlist mode in its + * current form. Just return NULL and move on. + */ + if (ring->buffer == NULL) + return NULL; + ipehr = I915_READ(RING_IPEHR(ring->mmio_base)); if (!ipehr_is_semaphore_wait(ring->dev, ipehr)) return NULL; @@ -3100,7 +3168,7 @@ static void vlv_display_irq_reset(struct drm_i915_private *dev_priv) { enum pipe pipe; - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); for_each_pipe(dev_priv, pipe) @@ -3194,74 +3262,124 @@ static void cherryview_irq_preinstall(struct drm_device *dev) vlv_display_irq_reset(dev_priv); } +static u32 intel_hpd_enabled_irqs(struct drm_device *dev, + const u32 hpd[HPD_NUM_PINS]) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_encoder *encoder; + u32 enabled_irqs = 0; + + for_each_intel_encoder(dev, encoder) + if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED) + enabled_irqs |= hpd[encoder->hpd_pin]; + + return enabled_irqs; +} + static void ibx_hpd_irq_setup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_encoder *intel_encoder; - u32 hotplug_irqs, hotplug, enabled_irqs = 0; + u32 hotplug_irqs, hotplug, enabled_irqs; if (HAS_PCH_IBX(dev)) { hotplug_irqs = SDE_HOTPLUG_MASK; - for_each_intel_encoder(dev, intel_encoder) - if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED) - enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin]; + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx); } else { hotplug_irqs = SDE_HOTPLUG_MASK_CPT; - for_each_intel_encoder(dev, intel_encoder) - if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED) - enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin]; + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt); } ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); /* * Enable digital hotplug on the PCH, and configure the DP short pulse - * duration to 2ms (which is the minimum in the Display Port spec) - * - * This register is the same on all known PCH chips. + * duration to 2ms (which is the minimum in the Display Port spec). + * The pulse duration bits are reserved on LPT+. */ hotplug = I915_READ(PCH_PORT_HOTPLUG); hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK); hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms; hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms; hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms; + /* + * When CPU and PCH are on the same package, port A + * HPD must be enabled in both north and south. + */ + if (HAS_PCH_LPT_LP(dev)) + hotplug |= PORTA_HOTPLUG_ENABLE; I915_WRITE(PCH_PORT_HOTPLUG, hotplug); } -static void bxt_hpd_irq_setup(struct drm_device *dev) +static void spt_hpd_irq_setup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_encoder *intel_encoder; - u32 hotplug_port = 0; - u32 hotplug_ctrl; - - /* Now, enable HPD */ - for_each_intel_encoder(dev, intel_encoder) { - if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark - == HPD_ENABLED) - hotplug_port |= hpd_bxt[intel_encoder->hpd_pin]; + u32 hotplug_irqs, hotplug, enabled_irqs; + + hotplug_irqs = SDE_HOTPLUG_MASK_SPT; + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt); + + ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); + + /* Enable digital hotplug on the PCH */ + hotplug = I915_READ(PCH_PORT_HOTPLUG); + hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE | + PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE; + I915_WRITE(PCH_PORT_HOTPLUG, hotplug); + + hotplug = I915_READ(PCH_PORT_HOTPLUG2); + hotplug |= PORTE_HOTPLUG_ENABLE; + I915_WRITE(PCH_PORT_HOTPLUG2, hotplug); +} + +static void ilk_hpd_irq_setup(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 hotplug_irqs, hotplug, enabled_irqs; + + if (INTEL_INFO(dev)->gen >= 8) { + hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG; + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw); + + bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); + } else if (INTEL_INFO(dev)->gen >= 7) { + hotplug_irqs = DE_DP_A_HOTPLUG_IVB; + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb); + + ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); + } else { + hotplug_irqs = DE_DP_A_HOTPLUG; + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk); + + ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); } - /* Mask all HPD control bits */ - hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK; + /* + * Enable digital hotplug on the CPU, and configure the DP short pulse + * duration to 2ms (which is the minimum in the Display Port spec) + * The pulse duration bits are reserved on HSW+. + */ + hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); + hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK; + hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms; + I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug); + + ibx_hpd_irq_setup(dev); +} + +static void bxt_hpd_irq_setup(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 hotplug_irqs, hotplug, enabled_irqs; - /* Enable requested port in hotplug control */ - /* TODO: implement (short) HPD support on port A */ - WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA); - if (hotplug_port & BXT_DE_PORT_HP_DDIB) - hotplug_ctrl |= BXT_DDIB_HPD_ENABLE; - if (hotplug_port & BXT_DE_PORT_HP_DDIC) - hotplug_ctrl |= BXT_DDIC_HPD_ENABLE; - I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl); + enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt); + hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK; - /* Unmask DDI hotplug in IMR */ - hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port; - I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl); + bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); - /* Enable DDI hotplug in IER */ - hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port; - I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl); - POSTING_READ(GEN8_DE_PORT_IER); + hotplug = I915_READ(PCH_PORT_HOTPLUG); + hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE | + PORTA_HOTPLUG_ENABLE; + I915_WRITE(PCH_PORT_HOTPLUG, hotplug); } static void ibx_irq_postinstall(struct drm_device *dev) @@ -3277,7 +3395,7 @@ static void ibx_irq_postinstall(struct drm_device *dev) else mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT; - GEN5_ASSERT_IIR_IS_ZERO(SDEIIR); + gen5_assert_iir_is_zero(dev_priv, SDEIIR); I915_WRITE(SDEIMR, ~mask); } @@ -3329,15 +3447,17 @@ static int ironlake_irq_postinstall(struct drm_device *dev) DE_PLANEB_FLIP_DONE_IVB | DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB); extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB | - DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB); + DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB | + DE_DP_A_HOTPLUG_IVB); } else { display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE | DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE | DE_POISON); - extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT | - DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN; + extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT | + DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN | + DE_DP_A_HOTPLUG); } dev_priv->irq_mask = ~display_mask; @@ -3359,7 +3479,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev) * setup is guaranteed to run in single-threaded context. But we * need it to make the assert_spin_locked happy. */ spin_lock_irq(&dev_priv->irq_lock); - ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT); + ilk_enable_display_irq(dev_priv, DE_PCU_EVENT); spin_unlock_irq(&dev_priv->irq_lock); } @@ -3464,7 +3584,7 @@ static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv) { dev_priv->irq_mask = ~0; - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); POSTING_READ(PORT_HOTPLUG_EN); I915_WRITE(VLV_IIR, 0xffffffff); @@ -3533,24 +3653,31 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) { uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; uint32_t de_pipe_enables; - int pipe; - u32 de_port_en = GEN8_AUX_CHANNEL_A; + u32 de_port_masked = GEN8_AUX_CHANNEL_A; + u32 de_port_enables; + enum pipe pipe; - if (IS_GEN9(dev_priv)) { + if (INTEL_INFO(dev_priv)->gen >= 9) { de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE | GEN9_DE_PIPE_IRQ_FAULT_ERRORS; - de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | - GEN9_AUX_CHANNEL_D; - + de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | + GEN9_AUX_CHANNEL_D; if (IS_BROXTON(dev_priv)) - de_port_en |= BXT_DE_PORT_GMBUS; - } else + de_port_masked |= BXT_DE_PORT_GMBUS; + } else { de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE | GEN8_DE_PIPE_IRQ_FAULT_ERRORS; + } de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; + de_port_enables = de_port_masked; + if (IS_BROXTON(dev_priv)) + de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK; + else if (IS_BROADWELL(dev_priv)) + de_port_enables |= GEN8_PORT_DP_A_HOTPLUG; + dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked; dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked; dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked; @@ -3562,7 +3689,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) dev_priv->de_irq_mask[pipe], de_pipe_enables); - GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en); + GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables); } static int gen8_irq_postinstall(struct drm_device *dev) @@ -3770,7 +3897,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); for_each_pipe(dev_priv, pipe) { - int reg = PIPESTAT(pipe); + i915_reg_t reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); /* @@ -3831,7 +3958,7 @@ static void i915_irq_preinstall(struct drm_device * dev) int pipe; if (I915_HAS_HOTPLUG(dev)) { - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); } @@ -3865,7 +3992,7 @@ static int i915_irq_postinstall(struct drm_device *dev) I915_USER_INTERRUPT; if (I915_HAS_HOTPLUG(dev)) { - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); POSTING_READ(PORT_HOTPLUG_EN); /* Enable in IER... */ @@ -3951,7 +4078,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); for_each_pipe(dev_priv, pipe) { - int reg = PIPESTAT(pipe); + i915_reg_t reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); /* Clear the PIPE*STAT regs before the IIR */ @@ -4027,7 +4154,7 @@ static void i915_irq_uninstall(struct drm_device * dev) int pipe; if (I915_HAS_HOTPLUG(dev)) { - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); } @@ -4048,7 +4175,7 @@ static void i965_irq_preinstall(struct drm_device * dev) struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); I915_WRITE(HWSTAM, 0xeffe); @@ -4109,7 +4236,7 @@ static int i965_irq_postinstall(struct drm_device *dev) I915_WRITE(IER, enable_mask); POSTING_READ(IER); - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); POSTING_READ(PORT_HOTPLUG_EN); i915_enable_asle_pipestat(dev); @@ -4120,29 +4247,27 @@ static int i965_irq_postinstall(struct drm_device *dev) static void i915_hpd_irq_setup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_encoder *intel_encoder; u32 hotplug_en; assert_spin_locked(&dev_priv->irq_lock); - hotplug_en = I915_READ(PORT_HOTPLUG_EN); - hotplug_en &= ~HOTPLUG_INT_EN_MASK; /* Note HDMI and DP share hotplug bits */ /* enable bits are the same for all generations */ - for_each_intel_encoder(dev, intel_encoder) - if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED) - hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin]; + hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915); /* Programming the CRT detection parameters tends to generate a spurious hotplug event about three seconds later. So just do it once. */ if (IS_G4X(dev)) hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; - hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK; hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; /* Ignore TV since it's buggy */ - I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); + i915_hotplug_interrupt_update_locked(dev_priv, + HOTPLUG_INT_EN_MASK | + CRT_HOTPLUG_VOLTAGE_COMPARE_MASK | + CRT_HOTPLUG_ACTIVATION_PERIOD_64, + hotplug_en); } static irqreturn_t i965_irq_handler(int irq, void *arg) @@ -4175,7 +4300,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) DRM_DEBUG("Command parser error, iir 0x%08x\n", iir); for_each_pipe(dev_priv, pipe) { - int reg = PIPESTAT(pipe); + i915_reg_t reg = PIPESTAT(pipe); pipe_stats[pipe] = I915_READ(reg); /* @@ -4255,7 +4380,7 @@ static void i965_irq_uninstall(struct drm_device * dev) if (!dev_priv) return; - I915_WRITE(PORT_HOTPLUG_EN, 0); + i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); I915_WRITE(HWSTAM, 0xffffffff); @@ -4270,46 +4395,6 @@ static void i965_irq_uninstall(struct drm_device * dev) I915_WRITE(IIR, I915_READ(IIR)); } -static void intel_hpd_irq_reenable_work(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), - hotplug_reenable_work.work); - struct drm_device *dev = dev_priv->dev; - struct drm_mode_config *mode_config = &dev->mode_config; - int i; - - intel_runtime_pm_get(dev_priv); - - spin_lock_irq(&dev_priv->irq_lock); - for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) { - struct drm_connector *connector; - - if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED) - continue; - - dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED; - - list_for_each_entry(connector, &mode_config->connector_list, head) { - struct intel_connector *intel_connector = to_intel_connector(connector); - - if (intel_connector->encoder->hpd_pin == i) { - if (connector->polled != intel_connector->polled) - DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n", - connector->name); - connector->polled = intel_connector->polled; - if (!connector->polled) - connector->polled = DRM_CONNECTOR_POLL_HPD; - } - } - } - if (dev_priv->display.hpd_irq_setup) - dev_priv->display.hpd_irq_setup(dev); - spin_unlock_irq(&dev_priv->irq_lock); - - intel_runtime_pm_put(dev_priv); -} - /** * intel_irq_init - initializes irq support * @dev_priv: i915 device instance @@ -4321,8 +4406,8 @@ void intel_irq_init(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; - INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func); - INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func); + intel_hpd_init_work(dev_priv); + INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); @@ -4335,8 +4420,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work, i915_hangcheck_elapsed); - INIT_DELAYED_WORK(&dev_priv->hotplug_reenable_work, - intel_hpd_irq_reenable_work); pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); @@ -4345,7 +4428,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev->driver->get_vblank_counter = i8xx_get_vblank_counter; } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) { dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */ - dev->driver->get_vblank_counter = gm45_get_vblank_counter; + dev->driver->get_vblank_counter = g4x_get_vblank_counter; } else { dev->driver->get_vblank_counter = i915_get_vblank_counter; dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ @@ -4385,10 +4468,12 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev->driver->irq_uninstall = gen8_irq_uninstall; dev->driver->enable_vblank = gen8_enable_vblank; dev->driver->disable_vblank = gen8_disable_vblank; - if (HAS_PCH_SPLIT(dev)) - dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup; - else + if (IS_BROXTON(dev)) dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; + else if (HAS_PCH_SPT(dev)) + dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; + else + dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; } else if (HAS_PCH_SPLIT(dev)) { dev->driver->irq_handler = ironlake_irq_handler; dev->driver->irq_preinstall = ironlake_irq_reset; @@ -4396,7 +4481,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev->driver->irq_uninstall = ironlake_irq_uninstall; dev->driver->enable_vblank = ironlake_enable_vblank; dev->driver->disable_vblank = ironlake_disable_vblank; - dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup; + dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; } else { if (INTEL_INFO(dev_priv)->gen == 2) { dev->driver->irq_preinstall = i8xx_irq_preinstall; @@ -4422,46 +4507,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) } /** - * intel_hpd_init - initializes and enables hpd support - * @dev_priv: i915 device instance - * - * This function enables the hotplug support. It requires that interrupts have - * already been enabled with intel_irq_init_hw(). From this point on hotplug and - * poll request can run concurrently to other code, so locking rules must be - * obeyed. - * - * This is a separate step from interrupt enabling to simplify the locking rules - * in the driver load and resume code. - */ -void intel_hpd_init(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct drm_mode_config *mode_config = &dev->mode_config; - struct drm_connector *connector; - int i; - - for (i = 1; i < HPD_NUM_PINS; i++) { - dev_priv->hpd_stats[i].hpd_cnt = 0; - dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED; - } - list_for_each_entry(connector, &mode_config->connector_list, head) { - struct intel_connector *intel_connector = to_intel_connector(connector); - connector->polled = intel_connector->polled; - if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE) - connector->polled = DRM_CONNECTOR_POLL_HPD; - if (intel_connector->mst_port) - connector->polled = DRM_CONNECTOR_POLL_HPD; - } - - /* Interrupt setup is already guaranteed to be single-threaded, this is - * just to make the assert_spin_locked checks happy. */ - spin_lock_irq(&dev_priv->irq_lock); - if (dev_priv->display.hpd_irq_setup) - dev_priv->display.hpd_irq_setup(dev); - spin_unlock_irq(&dev_priv->irq_lock); -} - -/** * intel_irq_install - enables the hardware interrupt * @dev_priv: i915 device instance * diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 8ac5a1b29ac0..835d6099c769 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -28,18 +28,18 @@ struct i915_params i915 __read_mostly = { .modeset = -1, .panel_ignore_lid = 1, .semaphores = -1, - .lvds_downclock = 0, .lvds_channel_mode = 0, .panel_use_ssc = -1, .vbt_sdvo_panel_type = -1, .enable_rc6 = -1, + .enable_dc = -1, .enable_fbc = -1, .enable_execlists = -1, .enable_hangcheck = true, .enable_ppgtt = -1, .enable_psr = 0, .preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT), - .disable_power_well = 1, + .disable_power_well = -1, .enable_ips = 1, .fastboot = 0, .prefault_disable = 0, @@ -54,14 +54,16 @@ struct i915_params i915 __read_mostly = { .verbose_state_checks = 1, .nuclear_pageflip = 0, .edp_vswing = 0, + .enable_guc_submission = false, + .guc_log_level = -1, }; module_param_named(modeset, i915.modeset, int, 0400); MODULE_PARM_DESC(modeset, - "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, " + "Use kernel modesetting [KMS] (0=disable, " "1=on, -1=force vga console preference [default])"); -module_param_named(panel_ignore_lid, i915.panel_ignore_lid, int, 0600); +module_param_named_unsafe(panel_ignore_lid, i915.panel_ignore_lid, int, 0600); MODULE_PARM_DESC(panel_ignore_lid, "Override lid status (0=autodetect, 1=autodetect disabled [default], " "-1=force lid closed, -2=force lid open)"); @@ -79,35 +81,35 @@ MODULE_PARM_DESC(enable_rc6, "For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. " "default: -1 (use per-chip default)"); +module_param_named_unsafe(enable_dc, i915.enable_dc, int, 0400); +MODULE_PARM_DESC(enable_dc, + "Enable power-saving display C-states. " + "(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6)"); + module_param_named_unsafe(enable_fbc, i915.enable_fbc, int, 0600); MODULE_PARM_DESC(enable_fbc, "Enable frame buffer compression for power savings " "(default: -1 (use per-chip default))"); -module_param_named(lvds_downclock, i915.lvds_downclock, int, 0400); -MODULE_PARM_DESC(lvds_downclock, - "Use panel (LVDS/eDP) downclocking for power savings " - "(default: false)"); - -module_param_named(lvds_channel_mode, i915.lvds_channel_mode, int, 0600); +module_param_named_unsafe(lvds_channel_mode, i915.lvds_channel_mode, int, 0600); MODULE_PARM_DESC(lvds_channel_mode, "Specify LVDS channel mode " "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)"); -module_param_named(lvds_use_ssc, i915.panel_use_ssc, int, 0600); +module_param_named_unsafe(lvds_use_ssc, i915.panel_use_ssc, int, 0600); MODULE_PARM_DESC(lvds_use_ssc, "Use Spread Spectrum Clock with panels [LVDS/eDP] " "(default: auto from VBT)"); -module_param_named(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0600); +module_param_named_unsafe(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0600); MODULE_PARM_DESC(vbt_sdvo_panel_type, "Override/Ignore selection of SDVO panel mode in the VBT " "(-2=ignore, -1=auto [default], index in VBT BIOS table)"); -module_param_named(reset, i915.reset, bool, 0600); +module_param_named_unsafe(reset, i915.reset, bool, 0600); MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)"); -module_param_named(enable_hangcheck, i915.enable_hangcheck, bool, 0644); +module_param_named_unsafe(enable_hangcheck, i915.enable_hangcheck, bool, 0644); MODULE_PARM_DESC(enable_hangcheck, "Periodically check GPU activity for detecting hangs. " "WARNING: Disabling this can cause system wide hangs. " @@ -116,25 +118,26 @@ MODULE_PARM_DESC(enable_hangcheck, module_param_named_unsafe(enable_ppgtt, i915.enable_ppgtt, int, 0400); MODULE_PARM_DESC(enable_ppgtt, "Override PPGTT usage. " - "(-1=auto [default], 0=disabled, 1=aliasing, 2=full)"); + "(-1=auto [default], 0=disabled, 1=aliasing, 2=full, 3=full with extended address space)"); -module_param_named(enable_execlists, i915.enable_execlists, int, 0400); +module_param_named_unsafe(enable_execlists, i915.enable_execlists, int, 0400); MODULE_PARM_DESC(enable_execlists, "Override execlists usage. " "(-1=auto [default], 0=disabled, 1=enabled)"); -module_param_named(enable_psr, i915.enable_psr, int, 0600); +module_param_named_unsafe(enable_psr, i915.enable_psr, int, 0600); MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)"); -module_param_named(preliminary_hw_support, i915.preliminary_hw_support, int, 0600); +module_param_named_unsafe(preliminary_hw_support, i915.preliminary_hw_support, int, 0600); MODULE_PARM_DESC(preliminary_hw_support, "Enable preliminary hardware support."); -module_param_named(disable_power_well, i915.disable_power_well, int, 0600); +module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400); MODULE_PARM_DESC(disable_power_well, - "Disable the power well when possible (default: true)"); + "Disable display power wells when possible " + "(-1=auto [default], 0=power wells always on, 1=power wells disabled when possible)"); -module_param_named(enable_ips, i915.enable_ips, int, 0600); +module_param_named_unsafe(enable_ips, i915.enable_ips, int, 0600); MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)"); module_param_named(fastboot, i915.fastboot, bool, 0600); @@ -151,7 +154,7 @@ MODULE_PARM_DESC(load_detect_test, "Force-enable the VGA load detect code for testing (default:false). " "For developers only."); -module_param_named(invert_brightness, i915.invert_brightness, int, 0600); +module_param_named_unsafe(invert_brightness, i915.invert_brightness, int, 0600); MODULE_PARM_DESC(invert_brightness, "Invert backlight brightness " "(-1 force normal, 0 machine defaults, 1 force inversion), please " @@ -162,14 +165,14 @@ MODULE_PARM_DESC(invert_brightness, module_param_named(disable_display, i915.disable_display, bool, 0600); MODULE_PARM_DESC(disable_display, "Disable display (default: false)"); -module_param_named(disable_vtd_wa, i915.disable_vtd_wa, bool, 0600); +module_param_named_unsafe(disable_vtd_wa, i915.disable_vtd_wa, bool, 0600); MODULE_PARM_DESC(disable_vtd_wa, "Disable all VT-d workarounds (default: false)"); -module_param_named(enable_cmd_parser, i915.enable_cmd_parser, int, 0600); +module_param_named_unsafe(enable_cmd_parser, i915.enable_cmd_parser, int, 0600); MODULE_PARM_DESC(enable_cmd_parser, "Enable command parsing (1=enabled [default], 0=disabled)"); -module_param_named(use_mmio_flip, i915.use_mmio_flip, int, 0600); +module_param_named_unsafe(use_mmio_flip, i915.use_mmio_flip, int, 0600); MODULE_PARM_DESC(use_mmio_flip, "use MMIO flips (-1=never, 0=driver discretion [default], 1=always)"); @@ -184,7 +187,7 @@ MODULE_PARM_DESC(verbose_state_checks, module_param_named_unsafe(nuclear_pageflip, i915.nuclear_pageflip, bool, 0600); MODULE_PARM_DESC(nuclear_pageflip, - "Force atomic modeset functionality; only planes work for now (default: false)."); + "Force atomic modeset functionality; asynchronous mode is not yet supported. (default: false)."); /* WA to get away with the default setting in VBT for early platforms.Will be removed */ module_param_named_unsafe(edp_vswing, i915.edp_vswing, int, 0400); @@ -192,3 +195,10 @@ MODULE_PARM_DESC(edp_vswing, "Ignore/Override vswing pre-emph table selection from VBT " "(0=use value from vbt [default], 1=low power swing(200mV)," "2=default swing(400mV))"); + +module_param_named_unsafe(enable_guc_submission, i915.enable_guc_submission, bool, 0400); +MODULE_PARM_DESC(enable_guc_submission, "Enable GuC submission (default:false)"); + +module_param_named(guc_log_level, i915.guc_log_level, int, 0400); +MODULE_PARM_DESC(guc_log_level, + "GuC firmware logging level (-1:disabled (default), 0-3:enabled)"); diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 2030f602cbf8..1dae5ac3e0b1 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -25,14 +25,43 @@ #ifndef _I915_REG_H_ #define _I915_REG_H_ +typedef struct { + uint32_t reg; +} i915_reg_t; + +#define _MMIO(r) ((const i915_reg_t){ .reg = (r) }) + +#define INVALID_MMIO_REG _MMIO(0) + +static inline uint32_t i915_mmio_reg_offset(i915_reg_t reg) +{ + return reg.reg; +} + +static inline bool i915_mmio_reg_equal(i915_reg_t a, i915_reg_t b) +{ + return i915_mmio_reg_offset(a) == i915_mmio_reg_offset(b); +} + +static inline bool i915_mmio_reg_valid(i915_reg_t reg) +{ + return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG); +} + #define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a))) +#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) #define _PLANE(plane, a, b) _PIPE(plane, a, b) -#define _TRANSCODER(tran, a, b) ((a) + (tran)*((b)-(a))) +#define _MMIO_PLANE(plane, a, b) _MMIO_PIPE(plane, a, b) +#define _TRANS(tran, a, b) ((a) + (tran)*((b)-(a))) +#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) #define _PORT(port, a, b) ((a) + (port)*((b)-(a))) +#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) #define _PIPE3(pipe, a, b, c) ((pipe) == PIPE_A ? (a) : \ (pipe) == PIPE_B ? (b) : (c)) +#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PIPE3(pipe, a, b, c)) #define _PORT3(port, a, b, c) ((port) == PORT_A ? (a) : \ (port) == PORT_B ? (b) : (c)) +#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PORT3(pipe, a, b, c)) #define _MASKED_FIELD(mask, value) ({ \ if (__builtin_constant_p(mask)) \ @@ -50,12 +79,17 @@ /* PCI config space */ -#define HPLLCC 0xc0 /* 855 only */ -#define GC_CLOCK_CONTROL_MASK (0xf << 0) +#define HPLLCC 0xc0 /* 85x only */ +#define GC_CLOCK_CONTROL_MASK (0x7 << 0) #define GC_CLOCK_133_200 (0 << 0) #define GC_CLOCK_100_200 (1 << 0) #define GC_CLOCK_100_133 (2 << 0) -#define GC_CLOCK_166_250 (3 << 0) +#define GC_CLOCK_133_266 (3 << 0) +#define GC_CLOCK_133_200_2 (4 << 0) +#define GC_CLOCK_133_266_2 (5 << 0) +#define GC_CLOCK_166_266 (6 << 0) +#define GC_CLOCK_166_250 (7 << 0) + #define GCFGC2 0xda #define GCFGC 0xf0 /* 915+ only */ #define GC_LOW_FREQUENCY_ENABLE (1 << 7) @@ -100,14 +134,14 @@ #define GRDOM_RESET_STATUS (1<<1) #define GRDOM_RESET_ENABLE (1<<0) -#define ILK_GDSR 0x2ca4 /* MCHBAR offset */ +#define ILK_GDSR _MMIO(MCHBAR_MIRROR_BASE + 0x2ca4) #define ILK_GRDOM_FULL (0<<1) #define ILK_GRDOM_RENDER (1<<1) #define ILK_GRDOM_MEDIA (3<<1) #define ILK_GRDOM_MASK (3<<1) #define ILK_GRDOM_RESET_ENABLE (1<<0) -#define GEN6_MBCUNIT_SNPCR 0x900c /* for LLC config */ +#define GEN6_MBCUNIT_SNPCR _MMIO(0x900c) /* for LLC config */ #define GEN6_MBC_SNPCR_SHIFT 21 #define GEN6_MBC_SNPCR_MASK (3<<21) #define GEN6_MBC_SNPCR_MAX (0<<21) @@ -115,31 +149,31 @@ #define GEN6_MBC_SNPCR_LOW (2<<21) #define GEN6_MBC_SNPCR_MIN (3<<21) /* only 1/16th of the cache is shared */ -#define VLV_G3DCTL 0x9024 -#define VLV_GSCKGCTL 0x9028 +#define VLV_G3DCTL _MMIO(0x9024) +#define VLV_GSCKGCTL _MMIO(0x9028) -#define GEN6_MBCTL 0x0907c +#define GEN6_MBCTL _MMIO(0x0907c) #define GEN6_MBCTL_ENABLE_BOOT_FETCH (1 << 4) #define GEN6_MBCTL_CTX_FETCH_NEEDED (1 << 3) #define GEN6_MBCTL_BME_UPDATE_ENABLE (1 << 2) #define GEN6_MBCTL_MAE_UPDATE_ENABLE (1 << 1) #define GEN6_MBCTL_BOOT_FETCH_MECH (1 << 0) -#define GEN6_GDRST 0x941c +#define GEN6_GDRST _MMIO(0x941c) #define GEN6_GRDOM_FULL (1 << 0) #define GEN6_GRDOM_RENDER (1 << 1) #define GEN6_GRDOM_MEDIA (1 << 2) #define GEN6_GRDOM_BLT (1 << 3) -#define RING_PP_DIR_BASE(ring) ((ring)->mmio_base+0x228) -#define RING_PP_DIR_BASE_READ(ring) ((ring)->mmio_base+0x518) -#define RING_PP_DIR_DCLV(ring) ((ring)->mmio_base+0x220) +#define RING_PP_DIR_BASE(ring) _MMIO((ring)->mmio_base+0x228) +#define RING_PP_DIR_BASE_READ(ring) _MMIO((ring)->mmio_base+0x518) +#define RING_PP_DIR_DCLV(ring) _MMIO((ring)->mmio_base+0x220) #define PP_DIR_DCLV_2G 0xffffffff -#define GEN8_RING_PDP_UDW(ring, n) ((ring)->mmio_base+0x270 + ((n) * 8 + 4)) -#define GEN8_RING_PDP_LDW(ring, n) ((ring)->mmio_base+0x270 + (n) * 8) +#define GEN8_RING_PDP_UDW(ring, n) _MMIO((ring)->mmio_base+0x270 + (n) * 8 + 4) +#define GEN8_RING_PDP_LDW(ring, n) _MMIO((ring)->mmio_base+0x270 + (n) * 8) -#define GEN8_R_PWR_CLK_STATE 0x20C8 +#define GEN8_R_PWR_CLK_STATE _MMIO(0x20C8) #define GEN8_RPCS_ENABLE (1 << 31) #define GEN8_RPCS_S_CNT_ENABLE (1 << 18) #define GEN8_RPCS_S_CNT_SHIFT 15 @@ -152,9 +186,10 @@ #define GEN8_RPCS_EU_MIN_SHIFT 0 #define GEN8_RPCS_EU_MIN_MASK (0xf << GEN8_RPCS_EU_MIN_SHIFT) -#define GAM_ECOCHK 0x4090 +#define GAM_ECOCHK _MMIO(0x4090) #define BDW_DISABLE_HDC_INVALIDATION (1<<25) #define ECOCHK_SNB_BIT (1<<10) +#define ECOCHK_DIS_TLB (1<<8) #define HSW_ECOCHK_ARB_PRIO_SOL (1<<6) #define ECOCHK_PPGTT_CACHE64B (0x3<<3) #define ECOCHK_PPGTT_CACHE4B (0x0<<3) @@ -164,27 +199,37 @@ #define ECOCHK_PPGTT_WT_HSW (0x2<<3) #define ECOCHK_PPGTT_WB_HSW (0x3<<3) -#define GAC_ECO_BITS 0x14090 +#define GAC_ECO_BITS _MMIO(0x14090) #define ECOBITS_SNB_BIT (1<<13) #define ECOBITS_PPGTT_CACHE64B (3<<8) #define ECOBITS_PPGTT_CACHE4B (0<<8) -#define GAB_CTL 0x24000 +#define GAB_CTL _MMIO(0x24000) #define GAB_CTL_CONT_AFTER_PAGEFAULT (1<<8) -#define GEN7_BIOS_RESERVED 0x1082C0 -#define GEN7_BIOS_RESERVED_1M (0 << 5) -#define GEN7_BIOS_RESERVED_256K (1 << 5) -#define GEN8_BIOS_RESERVED_SHIFT 7 -#define GEN7_BIOS_RESERVED_MASK 0x1 -#define GEN8_BIOS_RESERVED_MASK 0x3 - +#define GEN6_STOLEN_RESERVED _MMIO(0x1082C0) +#define GEN6_STOLEN_RESERVED_ADDR_MASK (0xFFF << 20) +#define GEN7_STOLEN_RESERVED_ADDR_MASK (0x3FFF << 18) +#define GEN6_STOLEN_RESERVED_SIZE_MASK (3 << 4) +#define GEN6_STOLEN_RESERVED_1M (0 << 4) +#define GEN6_STOLEN_RESERVED_512K (1 << 4) +#define GEN6_STOLEN_RESERVED_256K (2 << 4) +#define GEN6_STOLEN_RESERVED_128K (3 << 4) +#define GEN7_STOLEN_RESERVED_SIZE_MASK (1 << 5) +#define GEN7_STOLEN_RESERVED_1M (0 << 5) +#define GEN7_STOLEN_RESERVED_256K (1 << 5) +#define GEN8_STOLEN_RESERVED_SIZE_MASK (3 << 7) +#define GEN8_STOLEN_RESERVED_1M (0 << 7) +#define GEN8_STOLEN_RESERVED_2M (1 << 7) +#define GEN8_STOLEN_RESERVED_4M (2 << 7) +#define GEN8_STOLEN_RESERVED_8M (3 << 7) /* VGA stuff */ #define VGA_ST01_MDA 0x3ba #define VGA_ST01_CGA 0x3da +#define _VGA_MSR_WRITE _MMIO(0x3c2) #define VGA_MSR_WRITE 0x3c2 #define VGA_MSR_READ 0x3cc #define VGA_MSR_MEM_EN (1<<1) @@ -316,6 +361,8 @@ #define MI_RESTORE_EXT_STATE_EN (1<<2) #define MI_FORCE_RESTORE (1<<1) #define MI_RESTORE_INHIBIT (1<<0) +#define HSW_MI_RS_SAVE_STATE_EN (1<<3) +#define HSW_MI_RS_RESTORE_STATE_EN (1<<2) #define MI_SEMAPHORE_SIGNAL MI_INSTR(0x1b, 0) /* GEN8+ */ #define MI_SEMAPHORE_TARGET(engine) ((engine)<<15) #define MI_SEMAPHORE_WAIT MI_INSTR(0x1c, 2) /* GEN8+ */ @@ -335,8 +382,8 @@ */ #define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*(x)-1) #define MI_LRI_FORCE_POSTED (1<<12) -#define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*(x)-1) -#define MI_STORE_REGISTER_MEM_GEN8(x) MI_INSTR(0x24, 3*(x)-1) +#define MI_STORE_REGISTER_MEM MI_INSTR(0x24, 1) +#define MI_STORE_REGISTER_MEM_GEN8 MI_INSTR(0x24, 2) #define MI_SRM_LRM_GLOBAL_GTT (1<<22) #define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */ #define MI_FLUSH_DW_STORE_INDEX (1<<21) @@ -347,6 +394,8 @@ #define MI_INVALIDATE_BSD (1<<7) #define MI_FLUSH_DW_USE_GTT (1<<2) #define MI_FLUSH_DW_USE_PPGTT (0<<2) +#define MI_LOAD_REGISTER_MEM MI_INSTR(0x29, 1) +#define MI_LOAD_REGISTER_MEM_GEN8 MI_INSTR(0x29, 2) #define MI_BATCH_BUFFER MI_INSTR(0x30, 1) #define MI_BATCH_NON_SECURE (1) /* for snb/ivb/vlv this also means "batch in ppgtt" when ppgtt is enabled. */ @@ -356,11 +405,14 @@ #define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0) #define MI_BATCH_GTT (2<<6) /* aliased with (1<<7) on gen4 */ #define MI_BATCH_BUFFER_START_GEN8 MI_INSTR(0x31, 1) +#define MI_BATCH_RESOURCE_STREAMER (1<<10) -#define MI_PREDICATE_SRC0 (0x2400) -#define MI_PREDICATE_SRC1 (0x2408) +#define MI_PREDICATE_SRC0 _MMIO(0x2400) +#define MI_PREDICATE_SRC0_UDW _MMIO(0x2400 + 4) +#define MI_PREDICATE_SRC1 _MMIO(0x2408) +#define MI_PREDICATE_SRC1_UDW _MMIO(0x2408 + 4) -#define MI_PREDICATE_RESULT_2 (0x2214) +#define MI_PREDICATE_RESULT_2 _MMIO(0x2214) #define LOWER_SLICE_ENABLED (1<<0) #define LOWER_SLICE_DISABLED (0<<0) @@ -409,7 +461,8 @@ #define ASYNC_FLIP (1<<22) #define DISPLAY_PLANE_A (0<<20) #define DISPLAY_PLANE_B (1<<20) -#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|(len-2)) +#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2)) +#define PIPE_CONTROL_FLUSH_L3 (1<<27) #define PIPE_CONTROL_GLOBAL_GTT_IVB (1<<24) /* gen7+ */ #define PIPE_CONTROL_MMIO_WRITE (1<<23) #define PIPE_CONTROL_STORE_DATA_INDEX (1<<21) @@ -426,6 +479,7 @@ #define PIPE_CONTROL_INDIRECT_STATE_DISABLE (1<<9) #define PIPE_CONTROL_NOTIFY (1<<8) #define PIPE_CONTROL_FLUSH_ENABLE (1<<7) /* gen7+ */ +#define PIPE_CONTROL_DC_FLUSH_ENABLE (1<<5) #define PIPE_CONTROL_VF_CACHE_INVALIDATE (1<<4) #define PIPE_CONTROL_CONST_CACHE_INVALIDATE (1<<3) #define PIPE_CONTROL_STATE_CACHE_INVALIDATE (1<<2) @@ -449,7 +503,6 @@ #define MI_CLFLUSH MI_INSTR(0x27, 0) #define MI_REPORT_PERF_COUNT MI_INSTR(0x28, 0) #define MI_REPORT_PERF_COUNT_GGTT (1<<0) -#define MI_LOAD_REGISTER_MEM MI_INSTR(0x29, 0) #define MI_LOAD_REGISTER_REG MI_INSTR(0x2A, 0) #define MI_RS_STORE_DATA_IMM MI_INSTR(0x2B, 0) #define MI_LOAD_URB_MEM MI_INSTR(0x2C, 0) @@ -488,45 +541,61 @@ /* * Registers used only by the command parser */ -#define BCS_SWCTRL 0x22200 - -#define GPGPU_THREADS_DISPATCHED 0x2290 -#define HS_INVOCATION_COUNT 0x2300 -#define DS_INVOCATION_COUNT 0x2308 -#define IA_VERTICES_COUNT 0x2310 -#define IA_PRIMITIVES_COUNT 0x2318 -#define VS_INVOCATION_COUNT 0x2320 -#define GS_INVOCATION_COUNT 0x2328 -#define GS_PRIMITIVES_COUNT 0x2330 -#define CL_INVOCATION_COUNT 0x2338 -#define CL_PRIMITIVES_COUNT 0x2340 -#define PS_INVOCATION_COUNT 0x2348 -#define PS_DEPTH_COUNT 0x2350 +#define BCS_SWCTRL _MMIO(0x22200) + +#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290) +#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4) +#define HS_INVOCATION_COUNT _MMIO(0x2300) +#define HS_INVOCATION_COUNT_UDW _MMIO(0x2300 + 4) +#define DS_INVOCATION_COUNT _MMIO(0x2308) +#define DS_INVOCATION_COUNT_UDW _MMIO(0x2308 + 4) +#define IA_VERTICES_COUNT _MMIO(0x2310) +#define IA_VERTICES_COUNT_UDW _MMIO(0x2310 + 4) +#define IA_PRIMITIVES_COUNT _MMIO(0x2318) +#define IA_PRIMITIVES_COUNT_UDW _MMIO(0x2318 + 4) +#define VS_INVOCATION_COUNT _MMIO(0x2320) +#define VS_INVOCATION_COUNT_UDW _MMIO(0x2320 + 4) +#define GS_INVOCATION_COUNT _MMIO(0x2328) +#define GS_INVOCATION_COUNT_UDW _MMIO(0x2328 + 4) +#define GS_PRIMITIVES_COUNT _MMIO(0x2330) +#define GS_PRIMITIVES_COUNT_UDW _MMIO(0x2330 + 4) +#define CL_INVOCATION_COUNT _MMIO(0x2338) +#define CL_INVOCATION_COUNT_UDW _MMIO(0x2338 + 4) +#define CL_PRIMITIVES_COUNT _MMIO(0x2340) +#define CL_PRIMITIVES_COUNT_UDW _MMIO(0x2340 + 4) +#define PS_INVOCATION_COUNT _MMIO(0x2348) +#define PS_INVOCATION_COUNT_UDW _MMIO(0x2348 + 4) +#define PS_DEPTH_COUNT _MMIO(0x2350) +#define PS_DEPTH_COUNT_UDW _MMIO(0x2350 + 4) /* There are the 4 64-bit counter registers, one for each stream output */ -#define GEN7_SO_NUM_PRIMS_WRITTEN(n) (0x5200 + (n) * 8) +#define GEN7_SO_NUM_PRIMS_WRITTEN(n) _MMIO(0x5200 + (n) * 8) +#define GEN7_SO_NUM_PRIMS_WRITTEN_UDW(n) _MMIO(0x5200 + (n) * 8 + 4) -#define GEN7_SO_PRIM_STORAGE_NEEDED(n) (0x5240 + (n) * 8) +#define GEN7_SO_PRIM_STORAGE_NEEDED(n) _MMIO(0x5240 + (n) * 8) +#define GEN7_SO_PRIM_STORAGE_NEEDED_UDW(n) _MMIO(0x5240 + (n) * 8 + 4) -#define GEN7_3DPRIM_END_OFFSET 0x2420 -#define GEN7_3DPRIM_START_VERTEX 0x2430 -#define GEN7_3DPRIM_VERTEX_COUNT 0x2434 -#define GEN7_3DPRIM_INSTANCE_COUNT 0x2438 -#define GEN7_3DPRIM_START_INSTANCE 0x243C -#define GEN7_3DPRIM_BASE_VERTEX 0x2440 +#define GEN7_3DPRIM_END_OFFSET _MMIO(0x2420) +#define GEN7_3DPRIM_START_VERTEX _MMIO(0x2430) +#define GEN7_3DPRIM_VERTEX_COUNT _MMIO(0x2434) +#define GEN7_3DPRIM_INSTANCE_COUNT _MMIO(0x2438) +#define GEN7_3DPRIM_START_INSTANCE _MMIO(0x243C) +#define GEN7_3DPRIM_BASE_VERTEX _MMIO(0x2440) -#define OACONTROL 0x2360 +#define GEN7_GPGPU_DISPATCHDIMX _MMIO(0x2500) +#define GEN7_GPGPU_DISPATCHDIMY _MMIO(0x2504) +#define GEN7_GPGPU_DISPATCHDIMZ _MMIO(0x2508) + +#define OACONTROL _MMIO(0x2360) #define _GEN7_PIPEA_DE_LOAD_SL 0x70068 #define _GEN7_PIPEB_DE_LOAD_SL 0x71068 -#define GEN7_PIPE_DE_LOAD_SL(pipe) _PIPE(pipe, \ - _GEN7_PIPEA_DE_LOAD_SL, \ - _GEN7_PIPEB_DE_LOAD_SL) +#define GEN7_PIPE_DE_LOAD_SL(pipe) _MMIO_PIPE(pipe, _GEN7_PIPEA_DE_LOAD_SL, _GEN7_PIPEB_DE_LOAD_SL) /* * Reset registers */ -#define DEBUG_RESET_I830 0x6070 +#define DEBUG_RESET_I830 _MMIO(0x6070) #define DEBUG_RESET_FULL (1<<7) #define DEBUG_RESET_RENDER (1<<8) #define DEBUG_RESET_DISPLAY (1<<9) @@ -534,7 +603,7 @@ /* * IOSF sideband */ -#define VLV_IOSF_DOORBELL_REQ (VLV_DISPLAY_BASE + 0x2100) +#define VLV_IOSF_DOORBELL_REQ _MMIO(VLV_DISPLAY_BASE + 0x2100) #define IOSF_DEVFN_SHIFT 24 #define IOSF_OPCODE_SHIFT 16 #define IOSF_PORT_SHIFT 8 @@ -551,8 +620,8 @@ #define IOSF_PORT_CCU 0xA9 #define IOSF_PORT_GPS_CORE 0x48 #define IOSF_PORT_FLISDSI 0x1B -#define VLV_IOSF_DATA (VLV_DISPLAY_BASE + 0x2104) -#define VLV_IOSF_ADDR (VLV_DISPLAY_BASE + 0x2108) +#define VLV_IOSF_DATA _MMIO(VLV_DISPLAY_BASE + 0x2104) +#define VLV_IOSF_ADDR _MMIO(VLV_DISPLAY_BASE + 0x2108) /* See configdb bunit SB addr map */ #define BUNIT_REG_BISOC 0x11 @@ -584,6 +653,7 @@ /* See the PUNIT HAS v0.8 for the below bits */ enum punit_power_well { + /* These numbers are fixed and must match the position of the pw bits */ PUNIT_POWER_WELL_RENDER = 0, PUNIT_POWER_WELL_MEDIA = 1, PUNIT_POWER_WELL_DISP2D = 3, @@ -596,10 +666,12 @@ enum punit_power_well { PUNIT_POWER_WELL_DPIO_RX1 = 11, PUNIT_POWER_WELL_DPIO_CMN_D = 12, - PUNIT_POWER_WELL_NUM, + /* Not actual bit groups. Used as IDs for lookup_power_well() */ + PUNIT_POWER_WELL_ALWAYS_ON, }; enum skl_disp_power_wells { + /* These numbers are fixed and must match the position of the pw bits */ SKL_DISP_PW_MISC_IO, SKL_DISP_PW_DDI_A_E, SKL_DISP_PW_DDI_B, @@ -607,6 +679,10 @@ enum skl_disp_power_wells { SKL_DISP_PW_DDI_D, SKL_DISP_PW_1 = 14, SKL_DISP_PW_2, + + /* Not actual bit groups. Used as IDs for lookup_power_well() */ + SKL_DISP_PW_ALWAYS_ON, + SKL_DISP_PW_DC_OFF, }; #define SKL_POWER_WELL_STATE(pw) (1 << ((pw) * 2)) @@ -707,12 +783,13 @@ enum skl_disp_power_wells { #define DSI_PLL_N1_DIV_MASK (3 << 16) #define DSI_PLL_M1_DIV_SHIFT 0 #define DSI_PLL_M1_DIV_MASK (0x1ff << 0) +#define CCK_CZ_CLOCK_CONTROL 0x62 #define CCK_DISPLAY_CLOCK_CONTROL 0x6b -#define DISPLAY_TRUNK_FORCE_ON (1 << 17) -#define DISPLAY_TRUNK_FORCE_OFF (1 << 16) -#define DISPLAY_FREQUENCY_STATUS (0x1f << 8) -#define DISPLAY_FREQUENCY_STATUS_SHIFT 8 -#define DISPLAY_FREQUENCY_VALUES (0x1f << 0) +#define CCK_TRUNK_FORCE_ON (1 << 17) +#define CCK_TRUNK_FORCE_OFF (1 << 16) +#define CCK_FREQUENCY_STATUS (0x1f << 8) +#define CCK_FREQUENCY_STATUS_SHIFT 8 +#define CCK_FREQUENCY_VALUES (0x1f << 0) /** * DOC: DPIO @@ -778,35 +855,35 @@ enum skl_disp_power_wells { * * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is * digital port D (CHV) or port A (BXT). - */ -/* - * Dual channel PHY (VLV/CHV/BXT) - * --------------------------------- - * | CH0 | CH1 | - * | CMN/PLL/REF | CMN/PLL/REF | - * |---------------|---------------| Display PHY - * | PCS01 | PCS23 | PCS01 | PCS23 | - * |-------|-------|-------|-------| - * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| - * --------------------------------- - * | DDI0 | DDI1 | DP/HDMI ports - * --------------------------------- * - * Single channel PHY (CHV/BXT) - * ----------------- - * | CH0 | - * | CMN/PLL/REF | - * |---------------| Display PHY - * | PCS01 | PCS23 | - * |-------|-------| - * |TX0|TX1|TX2|TX3| - * ----------------- - * | DDI2 | DP/HDMI port - * ----------------- + * + * Dual channel PHY (VLV/CHV/BXT) + * --------------------------------- + * | CH0 | CH1 | + * | CMN/PLL/REF | CMN/PLL/REF | + * |---------------|---------------| Display PHY + * | PCS01 | PCS23 | PCS01 | PCS23 | + * |-------|-------|-------|-------| + * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| + * --------------------------------- + * | DDI0 | DDI1 | DP/HDMI ports + * --------------------------------- + * + * Single channel PHY (CHV/BXT) + * ----------------- + * | CH0 | + * | CMN/PLL/REF | + * |---------------| Display PHY + * | PCS01 | PCS23 | + * |-------|-------| + * |TX0|TX1|TX2|TX3| + * ----------------- + * | DDI2 | DP/HDMI port + * ----------------- */ #define DPIO_DEVFN 0 -#define DPIO_CTL (VLV_DISPLAY_BASE + 0x2110) +#define DPIO_CTL _MMIO(VLV_DISPLAY_BASE + 0x2110) #define DPIO_MODSEL1 (1<<3) /* if ref clk b == 27 */ #define DPIO_MODSEL0 (1<<2) /* if ref clk a == 27 */ #define DPIO_SFR_BYPASS (1<<1) @@ -1078,6 +1155,12 @@ enum skl_disp_power_wells { #define DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE 1 /* 1: coarse & 0 : fine */ #define CHV_PLL_DW9(ch) _PIPE(ch, _CHV_PLL_DW9_CH0, _CHV_PLL_DW9_CH1) +#define _CHV_CMN_DW0_CH0 0x8100 +#define DPIO_ALLDL_POWERDOWN_SHIFT_CH0 19 +#define DPIO_ANYDL_POWERDOWN_SHIFT_CH0 18 +#define DPIO_ALLDL_POWERDOWN (1 << 1) +#define DPIO_ANYDL_POWERDOWN (1 << 0) + #define _CHV_CMN_DW5_CH0 0x8114 #define CHV_BUFRIGHTENA1_DISABLE (0 << 20) #define CHV_BUFRIGHTENA1_NORMAL (1 << 20) @@ -1114,10 +1197,23 @@ enum skl_disp_power_wells { #define _CHV_CMN_DW19_CH0 0x814c #define _CHV_CMN_DW6_CH1 0x8098 +#define DPIO_ALLDL_POWERDOWN_SHIFT_CH1 30 /* CL2 DW6 only */ +#define DPIO_ANYDL_POWERDOWN_SHIFT_CH1 29 /* CL2 DW6 only */ +#define DPIO_DYNPWRDOWNEN_CH1 (1 << 28) /* CL2 DW6 only */ #define CHV_CMN_USEDCLKCHANNEL (1 << 13) + #define CHV_CMN_DW19(ch) _PIPE(ch, _CHV_CMN_DW19_CH0, _CHV_CMN_DW6_CH1) +#define CHV_CMN_DW28 0x8170 +#define DPIO_CL1POWERDOWNEN (1 << 23) +#define DPIO_DYNPWRDOWNEN_CH0 (1 << 22) +#define DPIO_SUS_CLK_CONFIG_ON (0 << 0) +#define DPIO_SUS_CLK_CONFIG_CLKREQ (1 << 0) +#define DPIO_SUS_CLK_CONFIG_GATE (2 << 0) +#define DPIO_SUS_CLK_CONFIG_GATE_CLKREQ (3 << 0) + #define CHV_CMN_DW30 0x8178 +#define DPIO_CL2_LDOFUSE_PWRENB (1 << 6) #define DPIO_LRC_BYPASS (1 << 3) #define _TXLANE(ch, lane, offset) ((ch ? 0x2400 : 0) + \ @@ -1140,9 +1236,9 @@ enum skl_disp_power_wells { #define DPIO_UPAR_SHIFT 30 /* BXT PHY registers */ -#define _BXT_PHY(phy, a, b) _PIPE((phy), (a), (b)) +#define _BXT_PHY(phy, a, b) _MMIO_PIPE((phy), (a), (b)) -#define BXT_P_CR_GT_DISP_PWRON 0x138090 +#define BXT_P_CR_GT_DISP_PWRON _MMIO(0x138090) #define GT_DISPLAY_POWER_ON(phy) (1 << (phy)) #define _PHY_CTL_FAMILY_EDP 0x64C80 @@ -1158,16 +1254,18 @@ enum skl_disp_power_wells { #define PORT_PLL_ENABLE (1 << 31) #define PORT_PLL_LOCK (1 << 30) #define PORT_PLL_REF_SEL (1 << 27) -#define BXT_PORT_PLL_ENABLE(port) _PORT(port, _PORT_PLL_A, _PORT_PLL_B) +#define BXT_PORT_PLL_ENABLE(port) _MMIO_PORT(port, _PORT_PLL_A, _PORT_PLL_B) #define _PORT_PLL_EBB_0_A 0x162034 #define _PORT_PLL_EBB_0_B 0x6C034 #define _PORT_PLL_EBB_0_C 0x6C340 -#define PORT_PLL_P1_MASK (0x07 << 13) -#define PORT_PLL_P1(x) ((x) << 13) -#define PORT_PLL_P2_MASK (0x1f << 8) -#define PORT_PLL_P2(x) ((x) << 8) -#define BXT_PORT_PLL_EBB_0(port) _PORT3(port, _PORT_PLL_EBB_0_A, \ +#define PORT_PLL_P1_SHIFT 13 +#define PORT_PLL_P1_MASK (0x07 << PORT_PLL_P1_SHIFT) +#define PORT_PLL_P1(x) ((x) << PORT_PLL_P1_SHIFT) +#define PORT_PLL_P2_SHIFT 8 +#define PORT_PLL_P2_MASK (0x1f << PORT_PLL_P2_SHIFT) +#define PORT_PLL_P2(x) ((x) << PORT_PLL_P2_SHIFT) +#define BXT_PORT_PLL_EBB_0(port) _MMIO_PORT3(port, _PORT_PLL_EBB_0_A, \ _PORT_PLL_EBB_0_B, \ _PORT_PLL_EBB_0_C) @@ -1176,7 +1274,7 @@ enum skl_disp_power_wells { #define _PORT_PLL_EBB_4_C 0x6C344 #define PORT_PLL_10BIT_CLK_ENABLE (1 << 13) #define PORT_PLL_RECALIBRATE (1 << 14) -#define BXT_PORT_PLL_EBB_4(port) _PORT3(port, _PORT_PLL_EBB_4_A, \ +#define BXT_PORT_PLL_EBB_4(port) _MMIO_PORT3(port, _PORT_PLL_EBB_4_A, \ _PORT_PLL_EBB_4_B, \ _PORT_PLL_EBB_4_C) @@ -1186,8 +1284,9 @@ enum skl_disp_power_wells { /* PORT_PLL_0_A */ #define PORT_PLL_M2_MASK 0xFF /* PORT_PLL_1_A */ -#define PORT_PLL_N_MASK (0x0F << 8) -#define PORT_PLL_N(x) ((x) << 8) +#define PORT_PLL_N_SHIFT 8 +#define PORT_PLL_N_MASK (0x0F << PORT_PLL_N_SHIFT) +#define PORT_PLL_N(x) ((x) << PORT_PLL_N_SHIFT) /* PORT_PLL_2_A */ #define PORT_PLL_M2_FRAC_MASK 0x3FFFFF /* PORT_PLL_3_A */ @@ -1201,15 +1300,17 @@ enum skl_disp_power_wells { /* PORT_PLL_8_A */ #define PORT_PLL_TARGET_CNT_MASK 0x3FF /* PORT_PLL_9_A */ -#define PORT_PLL_LOCK_THRESHOLD_MASK 0xe +#define PORT_PLL_LOCK_THRESHOLD_SHIFT 1 +#define PORT_PLL_LOCK_THRESHOLD_MASK (0x7 << PORT_PLL_LOCK_THRESHOLD_SHIFT) /* PORT_PLL_10_A */ #define PORT_PLL_DCO_AMP_OVR_EN_H (1<<27) +#define PORT_PLL_DCO_AMP_DEFAULT 15 #define PORT_PLL_DCO_AMP_MASK 0x3c00 -#define PORT_PLL_DCO_AMP(x) (x<<10) +#define PORT_PLL_DCO_AMP(x) ((x)<<10) #define _PORT_PLL_BASE(port) _PORT3(port, _PORT_PLL_0_A, \ _PORT_PLL_0_B, \ _PORT_PLL_0_C) -#define BXT_PORT_PLL(port, idx) (_PORT_PLL_BASE(port) + (idx) * 4) +#define BXT_PORT_PLL(port, idx) _MMIO(_PORT_PLL_BASE(port) + (idx) * 4) /* BXT PHY common lane registers */ #define _PORT_CL1CM_DW0_A 0x162000 @@ -1247,7 +1348,7 @@ enum skl_disp_power_wells { _PORT_CL1CM_DW30_A) /* Defined for PHY0 only */ -#define BXT_PORT_CL2CM_DW6_BC 0x6C358 +#define BXT_PORT_CL2CM_DW6_BC _MMIO(0x6C358) #define DW6_OLDO_DYN_PWR_DOWN_EN (1 << 28) /* BXT PHY Ref registers */ @@ -1287,10 +1388,10 @@ enum skl_disp_power_wells { #define _PORT_PCS_DW10_GRP_A 0x162C28 #define _PORT_PCS_DW10_GRP_B 0x6CC28 #define _PORT_PCS_DW10_GRP_C 0x6CE28 -#define BXT_PORT_PCS_DW10_LN01(port) _PORT3(port, _PORT_PCS_DW10_LN01_A, \ +#define BXT_PORT_PCS_DW10_LN01(port) _MMIO_PORT3(port, _PORT_PCS_DW10_LN01_A, \ _PORT_PCS_DW10_LN01_B, \ _PORT_PCS_DW10_LN01_C) -#define BXT_PORT_PCS_DW10_GRP(port) _PORT3(port, _PORT_PCS_DW10_GRP_A, \ +#define BXT_PORT_PCS_DW10_GRP(port) _MMIO_PORT3(port, _PORT_PCS_DW10_GRP_A, \ _PORT_PCS_DW10_GRP_B, \ _PORT_PCS_DW10_GRP_C) #define TX2_SWING_CALC_INIT (1 << 31) @@ -1307,13 +1408,13 @@ enum skl_disp_power_wells { #define _PORT_PCS_DW12_GRP_C 0x6CE30 #define LANESTAGGER_STRAP_OVRD (1 << 6) #define LANE_STAGGER_MASK 0x1F -#define BXT_PORT_PCS_DW12_LN01(port) _PORT3(port, _PORT_PCS_DW12_LN01_A, \ +#define BXT_PORT_PCS_DW12_LN01(port) _MMIO_PORT3(port, _PORT_PCS_DW12_LN01_A, \ _PORT_PCS_DW12_LN01_B, \ _PORT_PCS_DW12_LN01_C) -#define BXT_PORT_PCS_DW12_LN23(port) _PORT3(port, _PORT_PCS_DW12_LN23_A, \ +#define BXT_PORT_PCS_DW12_LN23(port) _MMIO_PORT3(port, _PORT_PCS_DW12_LN23_A, \ _PORT_PCS_DW12_LN23_B, \ _PORT_PCS_DW12_LN23_C) -#define BXT_PORT_PCS_DW12_GRP(port) _PORT3(port, _PORT_PCS_DW12_GRP_A, \ +#define BXT_PORT_PCS_DW12_GRP(port) _MMIO_PORT3(port, _PORT_PCS_DW12_GRP_A, \ _PORT_PCS_DW12_GRP_B, \ _PORT_PCS_DW12_GRP_C) @@ -1327,10 +1428,10 @@ enum skl_disp_power_wells { #define _PORT_TX_DW2_GRP_A 0x162D08 #define _PORT_TX_DW2_GRP_B 0x6CD08 #define _PORT_TX_DW2_GRP_C 0x6CF08 -#define BXT_PORT_TX_DW2_GRP(port) _PORT3(port, _PORT_TX_DW2_GRP_A, \ +#define BXT_PORT_TX_DW2_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW2_GRP_A, \ _PORT_TX_DW2_GRP_B, \ _PORT_TX_DW2_GRP_C) -#define BXT_PORT_TX_DW2_LN0(port) _PORT3(port, _PORT_TX_DW2_LN0_A, \ +#define BXT_PORT_TX_DW2_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW2_LN0_A, \ _PORT_TX_DW2_LN0_B, \ _PORT_TX_DW2_LN0_C) #define MARGIN_000_SHIFT 16 @@ -1344,13 +1445,14 @@ enum skl_disp_power_wells { #define _PORT_TX_DW3_GRP_A 0x162D0C #define _PORT_TX_DW3_GRP_B 0x6CD0C #define _PORT_TX_DW3_GRP_C 0x6CF0C -#define BXT_PORT_TX_DW3_GRP(port) _PORT3(port, _PORT_TX_DW3_GRP_A, \ +#define BXT_PORT_TX_DW3_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW3_GRP_A, \ _PORT_TX_DW3_GRP_B, \ _PORT_TX_DW3_GRP_C) -#define BXT_PORT_TX_DW3_LN0(port) _PORT3(port, _PORT_TX_DW3_LN0_A, \ +#define BXT_PORT_TX_DW3_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW3_LN0_A, \ _PORT_TX_DW3_LN0_B, \ _PORT_TX_DW3_LN0_C) -#define UNIQE_TRANGE_EN_METHOD (1 << 27) +#define SCALE_DCOMP_METHOD (1 << 26) +#define UNIQUE_TRANGE_EN_METHOD (1 << 27) #define _PORT_TX_DW4_LN0_A 0x162510 #define _PORT_TX_DW4_LN0_B 0x6C510 @@ -1358,10 +1460,10 @@ enum skl_disp_power_wells { #define _PORT_TX_DW4_GRP_A 0x162D10 #define _PORT_TX_DW4_GRP_B 0x6CD10 #define _PORT_TX_DW4_GRP_C 0x6CF10 -#define BXT_PORT_TX_DW4_LN0(port) _PORT3(port, _PORT_TX_DW4_LN0_A, \ +#define BXT_PORT_TX_DW4_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW4_LN0_A, \ _PORT_TX_DW4_LN0_B, \ _PORT_TX_DW4_LN0_C) -#define BXT_PORT_TX_DW4_GRP(port) _PORT3(port, _PORT_TX_DW4_GRP_A, \ +#define BXT_PORT_TX_DW4_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW4_GRP_A, \ _PORT_TX_DW4_GRP_B, \ _PORT_TX_DW4_GRP_C) #define DEEMPH_SHIFT 24 @@ -1372,16 +1474,34 @@ enum skl_disp_power_wells { #define _PORT_TX_DW14_LN0_C 0x6C938 #define LATENCY_OPTIM_SHIFT 30 #define LATENCY_OPTIM (1 << LATENCY_OPTIM_SHIFT) -#define BXT_PORT_TX_DW14_LN(port, lane) (_PORT3((port), _PORT_TX_DW14_LN0_A, \ +#define BXT_PORT_TX_DW14_LN(port, lane) _MMIO(_PORT3((port), _PORT_TX_DW14_LN0_A, \ _PORT_TX_DW14_LN0_B, \ _PORT_TX_DW14_LN0_C) + \ _BXT_LANE_OFFSET(lane)) +/* UAIMI scratch pad register 1 */ +#define UAIMI_SPR1 _MMIO(0x4F074) +/* SKL VccIO mask */ +#define SKL_VCCIO_MASK 0x1 +/* SKL balance leg register */ +#define DISPIO_CR_TX_BMU_CR0 _MMIO(0x6C00C) +/* I_boost values */ +#define BALANCE_LEG_SHIFT(port) (8+3*(port)) +#define BALANCE_LEG_MASK(port) (7<<(8+3*(port))) +/* Balance leg disable bits */ +#define BALANCE_LEG_DISABLE_SHIFT 23 + /* * Fence registers + * [0-7] @ 0x2000 gen2,gen3 + * [8-15] @ 0x3000 945,g33,pnv + * + * [0-15] @ 0x3000 gen4,gen5 + * + * [0-15] @ 0x100000 gen6,vlv,chv + * [0-31] @ 0x100000 gen7+ */ -#define FENCE_REG_830_0 0x2000 -#define FENCE_REG_945_8 0x3000 +#define FENCE_REG(i) _MMIO(0x2000 + (((i) & 8) << 9) + ((i) & 7) * 4) #define I830_FENCE_START_MASK 0x07f80000 #define I830_FENCE_TILING_Y_SHIFT 12 #define I830_FENCE_SIZE_BITS(size) ((ffs((size) >> 19) - 1) << 8) @@ -1394,19 +1514,21 @@ enum skl_disp_power_wells { #define I915_FENCE_START_MASK 0x0ff00000 #define I915_FENCE_SIZE_BITS(size) ((ffs((size) >> 20) - 1) << 8) -#define FENCE_REG_965_0 0x03000 +#define FENCE_REG_965_LO(i) _MMIO(0x03000 + (i) * 8) +#define FENCE_REG_965_HI(i) _MMIO(0x03000 + (i) * 8 + 4) #define I965_FENCE_PITCH_SHIFT 2 #define I965_FENCE_TILING_Y_SHIFT 1 #define I965_FENCE_REG_VALID (1<<0) #define I965_FENCE_MAX_PITCH_VAL 0x0400 -#define FENCE_REG_SANDYBRIDGE_0 0x100000 -#define SANDYBRIDGE_FENCE_PITCH_SHIFT 32 +#define FENCE_REG_GEN6_LO(i) _MMIO(0x100000 + (i) * 8) +#define FENCE_REG_GEN6_HI(i) _MMIO(0x100000 + (i) * 8 + 4) +#define GEN6_FENCE_PITCH_SHIFT 32 #define GEN7_FENCE_MAX_PITCH_VAL 0x0800 /* control register for cpu gtt access */ -#define TILECTL 0x101000 +#define TILECTL _MMIO(0x101000) #define TILECTL_SWZCTL (1 << 0) #define TILECTL_TLBPF (1 << 1) #define TILECTL_TLB_PREFETCH_DIS (1 << 2) @@ -1415,30 +1537,30 @@ enum skl_disp_power_wells { /* * Instruction and interrupt control regs */ -#define PGTBL_CTL 0x02020 +#define PGTBL_CTL _MMIO(0x02020) #define PGTBL_ADDRESS_LO_MASK 0xfffff000 /* bits [31:12] */ #define PGTBL_ADDRESS_HI_MASK 0x000000f0 /* bits [35:32] (gen4) */ -#define PGTBL_ER 0x02024 -#define PRB0_BASE (0x2030-0x30) -#define PRB1_BASE (0x2040-0x30) /* 830,gen3 */ -#define PRB2_BASE (0x2050-0x30) /* gen3 */ -#define SRB0_BASE (0x2100-0x30) /* gen2 */ -#define SRB1_BASE (0x2110-0x30) /* gen2 */ -#define SRB2_BASE (0x2120-0x30) /* 830 */ -#define SRB3_BASE (0x2130-0x30) /* 830 */ +#define PGTBL_ER _MMIO(0x02024) +#define PRB0_BASE (0x2030-0x30) +#define PRB1_BASE (0x2040-0x30) /* 830,gen3 */ +#define PRB2_BASE (0x2050-0x30) /* gen3 */ +#define SRB0_BASE (0x2100-0x30) /* gen2 */ +#define SRB1_BASE (0x2110-0x30) /* gen2 */ +#define SRB2_BASE (0x2120-0x30) /* 830 */ +#define SRB3_BASE (0x2130-0x30) /* 830 */ #define RENDER_RING_BASE 0x02000 #define BSD_RING_BASE 0x04000 #define GEN6_BSD_RING_BASE 0x12000 #define GEN8_BSD2_RING_BASE 0x1c000 #define VEBOX_RING_BASE 0x1a000 #define BLT_RING_BASE 0x22000 -#define RING_TAIL(base) ((base)+0x30) -#define RING_HEAD(base) ((base)+0x34) -#define RING_START(base) ((base)+0x38) -#define RING_CTL(base) ((base)+0x3c) -#define RING_SYNC_0(base) ((base)+0x40) -#define RING_SYNC_1(base) ((base)+0x44) -#define RING_SYNC_2(base) ((base)+0x48) +#define RING_TAIL(base) _MMIO((base)+0x30) +#define RING_HEAD(base) _MMIO((base)+0x34) +#define RING_START(base) _MMIO((base)+0x38) +#define RING_CTL(base) _MMIO((base)+0x3c) +#define RING_SYNC_0(base) _MMIO((base)+0x40) +#define RING_SYNC_1(base) _MMIO((base)+0x44) +#define RING_SYNC_2(base) _MMIO((base)+0x48) #define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE)) #define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE)) #define GEN6_RVESYNC (RING_SYNC_2(RENDER_RING_BASE)) @@ -1451,47 +1573,52 @@ enum skl_disp_power_wells { #define GEN6_VEBSYNC (RING_SYNC_0(VEBOX_RING_BASE)) #define GEN6_VERSYNC (RING_SYNC_1(VEBOX_RING_BASE)) #define GEN6_VEVSYNC (RING_SYNC_2(VEBOX_RING_BASE)) -#define GEN6_NOSYNC 0 -#define RING_PSMI_CTL(base) ((base)+0x50) -#define RING_MAX_IDLE(base) ((base)+0x54) -#define RING_HWS_PGA(base) ((base)+0x80) -#define RING_HWS_PGA_GEN6(base) ((base)+0x2080) - -#define HSW_GTT_CACHE_EN 0x4024 +#define GEN6_NOSYNC INVALID_MMIO_REG +#define RING_PSMI_CTL(base) _MMIO((base)+0x50) +#define RING_MAX_IDLE(base) _MMIO((base)+0x54) +#define RING_HWS_PGA(base) _MMIO((base)+0x80) +#define RING_HWS_PGA_GEN6(base) _MMIO((base)+0x2080) +#define RING_RESET_CTL(base) _MMIO((base)+0xd0) +#define RESET_CTL_REQUEST_RESET (1 << 0) +#define RESET_CTL_READY_TO_RESET (1 << 1) + +#define HSW_GTT_CACHE_EN _MMIO(0x4024) #define GTT_CACHE_EN_ALL 0xF0007FFF -#define GEN7_WR_WATERMARK 0x4028 -#define GEN7_GFX_PRIO_CTRL 0x402C -#define ARB_MODE 0x4030 +#define GEN7_WR_WATERMARK _MMIO(0x4028) +#define GEN7_GFX_PRIO_CTRL _MMIO(0x402C) +#define ARB_MODE _MMIO(0x4030) #define ARB_MODE_SWIZZLE_SNB (1<<4) #define ARB_MODE_SWIZZLE_IVB (1<<5) -#define GEN7_GFX_PEND_TLB0 0x4034 -#define GEN7_GFX_PEND_TLB1 0x4038 +#define GEN7_GFX_PEND_TLB0 _MMIO(0x4034) +#define GEN7_GFX_PEND_TLB1 _MMIO(0x4038) /* L3, CVS, ZTLB, RCC, CASC LRA min, max values */ -#define GEN7_LRA_LIMITS_BASE 0x403C +#define GEN7_LRA_LIMITS(i) _MMIO(0x403C + (i) * 4) #define GEN7_LRA_LIMITS_REG_NUM 13 -#define GEN7_MEDIA_MAX_REQ_COUNT 0x4070 -#define GEN7_GFX_MAX_REQ_COUNT 0x4074 +#define GEN7_MEDIA_MAX_REQ_COUNT _MMIO(0x4070) +#define GEN7_GFX_MAX_REQ_COUNT _MMIO(0x4074) -#define GAMTARBMODE 0x04a08 +#define GAMTARBMODE _MMIO(0x04a08) #define ARB_MODE_BWGTLB_DISABLE (1<<9) #define ARB_MODE_SWIZZLE_BDW (1<<1) -#define RENDER_HWS_PGA_GEN7 (0x04080) -#define RING_FAULT_REG(ring) (0x4094 + 0x100*(ring)->id) +#define RENDER_HWS_PGA_GEN7 _MMIO(0x04080) +#define RING_FAULT_REG(ring) _MMIO(0x4094 + 0x100*(ring)->id) #define RING_FAULT_GTTSEL_MASK (1<<11) -#define RING_FAULT_SRCID(x) ((x >> 3) & 0xff) -#define RING_FAULT_FAULT_TYPE(x) ((x >> 1) & 0x3) +#define RING_FAULT_SRCID(x) (((x) >> 3) & 0xff) +#define RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3) #define RING_FAULT_VALID (1<<0) -#define DONE_REG 0x40b0 -#define GEN8_PRIVATE_PAT 0x40e0 -#define BSD_HWS_PGA_GEN7 (0x04180) -#define BLT_HWS_PGA_GEN7 (0x04280) -#define VEBOX_HWS_PGA_GEN7 (0x04380) -#define RING_ACTHD(base) ((base)+0x74) -#define RING_ACTHD_UDW(base) ((base)+0x5c) -#define RING_NOPID(base) ((base)+0x94) -#define RING_IMR(base) ((base)+0xa8) -#define RING_HWSTAM(base) ((base)+0x98) -#define RING_TIMESTAMP(base) ((base)+0x358) +#define DONE_REG _MMIO(0x40b0) +#define GEN8_PRIVATE_PAT_LO _MMIO(0x40e0) +#define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4) +#define BSD_HWS_PGA_GEN7 _MMIO(0x04180) +#define BLT_HWS_PGA_GEN7 _MMIO(0x04280) +#define VEBOX_HWS_PGA_GEN7 _MMIO(0x04380) +#define RING_ACTHD(base) _MMIO((base)+0x74) +#define RING_ACTHD_UDW(base) _MMIO((base)+0x5c) +#define RING_NOPID(base) _MMIO((base)+0x94) +#define RING_IMR(base) _MMIO((base)+0xa8) +#define RING_HWSTAM(base) _MMIO((base)+0x98) +#define RING_TIMESTAMP(base) _MMIO((base)+0x358) +#define RING_TIMESTAMP_UDW(base) _MMIO((base)+0x358 + 4) #define TAIL_ADDR 0x001FFFF8 #define HEAD_WRAP_COUNT 0xFFE00000 #define HEAD_WRAP_ONE 0x00200000 @@ -1508,54 +1635,65 @@ enum skl_disp_power_wells { #define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */ #define RING_WAIT_SEMAPHORE (1<<10) /* gen6+ */ -#define GEN7_TLB_RD_ADDR 0x4700 +#define GEN7_TLB_RD_ADDR _MMIO(0x4700) #if 0 -#define PRB0_TAIL 0x02030 -#define PRB0_HEAD 0x02034 -#define PRB0_START 0x02038 -#define PRB0_CTL 0x0203c -#define PRB1_TAIL 0x02040 /* 915+ only */ -#define PRB1_HEAD 0x02044 /* 915+ only */ -#define PRB1_START 0x02048 /* 915+ only */ -#define PRB1_CTL 0x0204c /* 915+ only */ +#define PRB0_TAIL _MMIO(0x2030) +#define PRB0_HEAD _MMIO(0x2034) +#define PRB0_START _MMIO(0x2038) +#define PRB0_CTL _MMIO(0x203c) +#define PRB1_TAIL _MMIO(0x2040) /* 915+ only */ +#define PRB1_HEAD _MMIO(0x2044) /* 915+ only */ +#define PRB1_START _MMIO(0x2048) /* 915+ only */ +#define PRB1_CTL _MMIO(0x204c) /* 915+ only */ #endif -#define IPEIR_I965 0x02064 -#define IPEHR_I965 0x02068 -#define INSTDONE_I965 0x0206c -#define GEN7_INSTDONE_1 0x0206c -#define GEN7_SC_INSTDONE 0x07100 -#define GEN7_SAMPLER_INSTDONE 0x0e160 -#define GEN7_ROW_INSTDONE 0x0e164 +#define IPEIR_I965 _MMIO(0x2064) +#define IPEHR_I965 _MMIO(0x2068) +#define GEN7_SC_INSTDONE _MMIO(0x7100) +#define GEN7_SAMPLER_INSTDONE _MMIO(0xe160) +#define GEN7_ROW_INSTDONE _MMIO(0xe164) #define I915_NUM_INSTDONE_REG 4 -#define RING_IPEIR(base) ((base)+0x64) -#define RING_IPEHR(base) ((base)+0x68) -#define RING_INSTDONE(base) ((base)+0x6c) -#define RING_INSTPS(base) ((base)+0x70) -#define RING_DMA_FADD(base) ((base)+0x78) -#define RING_DMA_FADD_UDW(base) ((base)+0x60) /* gen8+ */ -#define RING_INSTPM(base) ((base)+0xc0) -#define RING_MI_MODE(base) ((base)+0x9c) -#define INSTPS 0x02070 /* 965+ only */ -#define INSTDONE1 0x0207c /* 965+ only */ -#define ACTHD_I965 0x02074 -#define HWS_PGA 0x02080 +#define RING_IPEIR(base) _MMIO((base)+0x64) +#define RING_IPEHR(base) _MMIO((base)+0x68) +/* + * On GEN4, only the render ring INSTDONE exists and has a different + * layout than the GEN7+ version. + * The GEN2 counterpart of this register is GEN2_INSTDONE. + */ +#define RING_INSTDONE(base) _MMIO((base)+0x6c) +#define RING_INSTPS(base) _MMIO((base)+0x70) +#define RING_DMA_FADD(base) _MMIO((base)+0x78) +#define RING_DMA_FADD_UDW(base) _MMIO((base)+0x60) /* gen8+ */ +#define RING_INSTPM(base) _MMIO((base)+0xc0) +#define RING_MI_MODE(base) _MMIO((base)+0x9c) +#define INSTPS _MMIO(0x2070) /* 965+ only */ +#define GEN4_INSTDONE1 _MMIO(0x207c) /* 965+ only, aka INSTDONE_2 on SNB */ +#define ACTHD_I965 _MMIO(0x2074) +#define HWS_PGA _MMIO(0x2080) #define HWS_ADDRESS_MASK 0xfffff000 #define HWS_START_ADDRESS_SHIFT 4 -#define PWRCTXA 0x2088 /* 965GM+ only */ +#define PWRCTXA _MMIO(0x2088) /* 965GM+ only */ #define PWRCTX_EN (1<<0) -#define IPEIR 0x02088 -#define IPEHR 0x0208c -#define INSTDONE 0x02090 -#define NOPID 0x02094 -#define HWSTAM 0x02098 -#define DMA_FADD_I8XX 0x020d0 -#define RING_BBSTATE(base) ((base)+0x110) -#define RING_BBADDR(base) ((base)+0x140) -#define RING_BBADDR_UDW(base) ((base)+0x168) /* gen8+ */ - -#define ERROR_GEN6 0x040a0 -#define GEN7_ERR_INT 0x44040 +#define IPEIR _MMIO(0x2088) +#define IPEHR _MMIO(0x208c) +#define GEN2_INSTDONE _MMIO(0x2090) +#define NOPID _MMIO(0x2094) +#define HWSTAM _MMIO(0x2098) +#define DMA_FADD_I8XX _MMIO(0x20d0) +#define RING_BBSTATE(base) _MMIO((base)+0x110) +#define RING_BB_PPGTT (1 << 5) +#define RING_SBBADDR(base) _MMIO((base)+0x114) /* hsw+ */ +#define RING_SBBSTATE(base) _MMIO((base)+0x118) /* hsw+ */ +#define RING_SBBADDR_UDW(base) _MMIO((base)+0x11c) /* gen8+ */ +#define RING_BBADDR(base) _MMIO((base)+0x140) +#define RING_BBADDR_UDW(base) _MMIO((base)+0x168) /* gen8+ */ +#define RING_BB_PER_CTX_PTR(base) _MMIO((base)+0x1c0) /* gen8+ */ +#define RING_INDIRECT_CTX(base) _MMIO((base)+0x1c4) /* gen8+ */ +#define RING_INDIRECT_CTX_OFFSET(base) _MMIO((base)+0x1c8) /* gen8+ */ +#define RING_CTX_TIMESTAMP(base) _MMIO((base)+0x3a8) /* gen8+ */ + +#define ERROR_GEN6 _MMIO(0x40a0) +#define GEN7_ERR_INT _MMIO(0x44040) #define ERR_INT_POISON (1<<31) #define ERR_INT_MMIO_UNCLAIMED (1<<13) #define ERR_INT_PIPE_CRC_DONE_C (1<<8) @@ -1563,17 +1701,17 @@ enum skl_disp_power_wells { #define ERR_INT_PIPE_CRC_DONE_B (1<<5) #define ERR_INT_FIFO_UNDERRUN_B (1<<3) #define ERR_INT_PIPE_CRC_DONE_A (1<<2) -#define ERR_INT_PIPE_CRC_DONE(pipe) (1<<(2 + pipe*3)) +#define ERR_INT_PIPE_CRC_DONE(pipe) (1<<(2 + (pipe)*3)) #define ERR_INT_FIFO_UNDERRUN_A (1<<0) -#define ERR_INT_FIFO_UNDERRUN(pipe) (1<<(pipe*3)) +#define ERR_INT_FIFO_UNDERRUN(pipe) (1<<((pipe)*3)) -#define GEN8_FAULT_TLB_DATA0 0x04b10 -#define GEN8_FAULT_TLB_DATA1 0x04b14 +#define GEN8_FAULT_TLB_DATA0 _MMIO(0x4b10) +#define GEN8_FAULT_TLB_DATA1 _MMIO(0x4b14) -#define FPGA_DBG 0x42300 +#define FPGA_DBG _MMIO(0x42300) #define FPGA_DBG_RM_NOCLAIM (1<<31) -#define DERRMR 0x44050 +#define DERRMR _MMIO(0x44050) /* Note that HBLANK events are reserved on bdw+ */ #define DERRMR_PIPEA_SCANLINE (1<<0) #define DERRMR_PIPEA_PRI_FLIP_DONE (1<<1) @@ -1597,81 +1735,88 @@ enum skl_disp_power_wells { * for various sorts of correct behavior. The top 16 bits of each are * the enables for writing to the corresponding low bit. */ -#define _3D_CHICKEN 0x02084 +#define _3D_CHICKEN _MMIO(0x2084) #define _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB (1 << 10) -#define _3D_CHICKEN2 0x0208c +#define _3D_CHICKEN2 _MMIO(0x208c) /* Disables pipelining of read flushes past the SF-WIZ interface. * Required on all Ironlake steppings according to the B-Spec, but the * particular danger of not doing so is not specified. */ # define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14) -#define _3D_CHICKEN3 0x02090 +#define _3D_CHICKEN3 _MMIO(0x2090) #define _3D_CHICKEN_SF_DISABLE_OBJEND_CULL (1 << 10) #define _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL (1 << 5) #define _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x) ((x)<<1) /* gen8+ */ #define _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH (1 << 1) /* gen6 */ -#define MI_MODE 0x0209c +#define MI_MODE _MMIO(0x209c) # define VS_TIMER_DISPATCH (1 << 6) # define MI_FLUSH_ENABLE (1 << 12) # define ASYNC_FLIP_PERF_DISABLE (1 << 14) # define MODE_IDLE (1 << 9) # define STOP_RING (1 << 8) -#define GEN6_GT_MODE 0x20d0 -#define GEN7_GT_MODE 0x7008 +#define GEN6_GT_MODE _MMIO(0x20d0) +#define GEN7_GT_MODE _MMIO(0x7008) #define GEN6_WIZ_HASHING(hi, lo) (((hi) << 9) | ((lo) << 7)) #define GEN6_WIZ_HASHING_8x8 GEN6_WIZ_HASHING(0, 0) #define GEN6_WIZ_HASHING_8x4 GEN6_WIZ_HASHING(0, 1) #define GEN6_WIZ_HASHING_16x4 GEN6_WIZ_HASHING(1, 0) #define GEN6_WIZ_HASHING_MASK GEN6_WIZ_HASHING(1, 1) #define GEN6_TD_FOUR_ROW_DISPATCH_DISABLE (1 << 5) -#define GEN9_IZ_HASHING_MASK(slice) (0x3 << (slice * 2)) -#define GEN9_IZ_HASHING(slice, val) ((val) << (slice * 2)) +#define GEN9_IZ_HASHING_MASK(slice) (0x3 << ((slice) * 2)) +#define GEN9_IZ_HASHING(slice, val) ((val) << ((slice) * 2)) -#define GFX_MODE 0x02520 -#define GFX_MODE_GEN7 0x0229c -#define RING_MODE_GEN7(ring) ((ring)->mmio_base+0x29c) +#define GFX_MODE _MMIO(0x2520) +#define GFX_MODE_GEN7 _MMIO(0x229c) +#define RING_MODE_GEN7(ring) _MMIO((ring)->mmio_base+0x29c) #define GFX_RUN_LIST_ENABLE (1<<15) +#define GFX_INTERRUPT_STEERING (1<<14) #define GFX_TLB_INVALIDATE_EXPLICIT (1<<13) #define GFX_SURFACE_FAULT_ENABLE (1<<12) #define GFX_REPLAY_MODE (1<<11) #define GFX_PSMI_GRANULARITY (1<<10) #define GFX_PPGTT_ENABLE (1<<9) +#define GEN8_GFX_PPGTT_48B (1<<7) + +#define GFX_FORWARD_VBLANK_MASK (3<<5) +#define GFX_FORWARD_VBLANK_NEVER (0<<5) +#define GFX_FORWARD_VBLANK_ALWAYS (1<<5) +#define GFX_FORWARD_VBLANK_COND (2<<5) #define VLV_DISPLAY_BASE 0x180000 #define VLV_MIPI_BASE VLV_DISPLAY_BASE -#define VLV_GU_CTL0 (VLV_DISPLAY_BASE + 0x2030) -#define VLV_GU_CTL1 (VLV_DISPLAY_BASE + 0x2034) -#define SCPD0 0x0209c /* 915+ only */ -#define IER 0x020a0 -#define IIR 0x020a4 -#define IMR 0x020a8 -#define ISR 0x020ac -#define VLV_GUNIT_CLOCK_GATE (VLV_DISPLAY_BASE + 0x2060) +#define VLV_GU_CTL0 _MMIO(VLV_DISPLAY_BASE + 0x2030) +#define VLV_GU_CTL1 _MMIO(VLV_DISPLAY_BASE + 0x2034) +#define SCPD0 _MMIO(0x209c) /* 915+ only */ +#define IER _MMIO(0x20a0) +#define IIR _MMIO(0x20a4) +#define IMR _MMIO(0x20a8) +#define ISR _MMIO(0x20ac) +#define VLV_GUNIT_CLOCK_GATE _MMIO(VLV_DISPLAY_BASE + 0x2060) #define GINT_DIS (1<<22) #define GCFG_DIS (1<<8) -#define VLV_GUNIT_CLOCK_GATE2 (VLV_DISPLAY_BASE + 0x2064) -#define VLV_IIR_RW (VLV_DISPLAY_BASE + 0x2084) -#define VLV_IER (VLV_DISPLAY_BASE + 0x20a0) -#define VLV_IIR (VLV_DISPLAY_BASE + 0x20a4) -#define VLV_IMR (VLV_DISPLAY_BASE + 0x20a8) -#define VLV_ISR (VLV_DISPLAY_BASE + 0x20ac) -#define VLV_PCBR (VLV_DISPLAY_BASE + 0x2120) +#define VLV_GUNIT_CLOCK_GATE2 _MMIO(VLV_DISPLAY_BASE + 0x2064) +#define VLV_IIR_RW _MMIO(VLV_DISPLAY_BASE + 0x2084) +#define VLV_IER _MMIO(VLV_DISPLAY_BASE + 0x20a0) +#define VLV_IIR _MMIO(VLV_DISPLAY_BASE + 0x20a4) +#define VLV_IMR _MMIO(VLV_DISPLAY_BASE + 0x20a8) +#define VLV_ISR _MMIO(VLV_DISPLAY_BASE + 0x20ac) +#define VLV_PCBR _MMIO(VLV_DISPLAY_BASE + 0x2120) #define VLV_PCBR_ADDR_SHIFT 12 #define DISPLAY_PLANE_FLIP_PENDING(plane) (1<<(11-(plane))) /* A and B only */ -#define EIR 0x020b0 -#define EMR 0x020b4 -#define ESR 0x020b8 +#define EIR _MMIO(0x20b0) +#define EMR _MMIO(0x20b4) +#define ESR _MMIO(0x20b8) #define GM45_ERROR_PAGE_TABLE (1<<5) #define GM45_ERROR_MEM_PRIV (1<<4) #define I915_ERROR_PAGE_TABLE (1<<4) #define GM45_ERROR_CP_PRIV (1<<3) #define I915_ERROR_MEMORY_REFRESH (1<<1) #define I915_ERROR_INSTRUCTION (1<<0) -#define INSTPM 0x020c0 +#define INSTPM _MMIO(0x20c0) #define INSTPM_SELF_EN (1<<12) /* 915GM only */ #define INSTPM_AGPBUSY_INT_EN (1<<11) /* gen3: when disabled, pending interrupts will not assert AGPBUSY# and will only @@ -1679,14 +1824,14 @@ enum skl_disp_power_wells { #define INSTPM_FORCE_ORDERING (1<<7) /* GEN6+ */ #define INSTPM_TLB_INVALIDATE (1<<9) #define INSTPM_SYNC_FLUSH (1<<5) -#define ACTHD 0x020c8 -#define MEM_MODE 0x020cc +#define ACTHD _MMIO(0x20c8) +#define MEM_MODE _MMIO(0x20cc) #define MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1<<3) /* 830 only */ #define MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1<<2) /* 830/845 only */ #define MEM_DISPLAY_TRICKLE_FEED_DISABLE (1<<2) /* 85x only */ -#define FW_BLC 0x020d8 -#define FW_BLC2 0x020dc -#define FW_BLC_SELF 0x020e0 /* 915+ only */ +#define FW_BLC _MMIO(0x20d8) +#define FW_BLC2 _MMIO(0x20dc) +#define FW_BLC_SELF _MMIO(0x20e0) /* 915+ only */ #define FW_BLC_SELF_EN_MASK (1<<31) #define FW_BLC_SELF_FIFO_MASK (1<<16) /* 945 only */ #define FW_BLC_SELF_EN (1<<15) /* 945 only */ @@ -1694,7 +1839,7 @@ enum skl_disp_power_wells { #define MM_FIFO_WATERMARK 0x0001F000 #define LM_BURST_LENGTH 0x00000700 #define LM_FIFO_WATERMARK 0x0000001F -#define MI_ARB_STATE 0x020e4 /* 915+ only */ +#define MI_ARB_STATE _MMIO(0x20e4) /* 915+ only */ /* Make render/texture TLB fetches lower priorty than associated data * fetches. This is not turned on by default @@ -1758,11 +1903,11 @@ enum skl_disp_power_wells { #define MI_ARB_DISPLAY_PRIORITY_A_B (0 << 0) /* display A > display B */ #define MI_ARB_DISPLAY_PRIORITY_B_A (1 << 0) /* display B > display A */ -#define MI_STATE 0x020e4 /* gen2 only */ +#define MI_STATE _MMIO(0x20e4) /* gen2 only */ #define MI_AGPBUSY_INT_EN (1 << 1) /* 85x only */ #define MI_AGPBUSY_830_MODE (1 << 0) /* 85x only */ -#define CACHE_MODE_0 0x02120 /* 915+ only */ +#define CACHE_MODE_0 _MMIO(0x2120) /* 915+ only */ #define CM0_PIPELINED_RENDER_FLUSH_DISABLE (1<<8) #define CM0_IZ_OPT_DISABLE (1<<6) #define CM0_ZR_OPT_DISABLE (1<<5) @@ -1771,32 +1916,32 @@ enum skl_disp_power_wells { #define CM0_COLOR_EVICT_DISABLE (1<<3) #define CM0_DEPTH_WRITE_DISABLE (1<<1) #define CM0_RC_OP_FLUSH_DISABLE (1<<0) -#define GFX_FLSH_CNTL 0x02170 /* 915+ only */ -#define GFX_FLSH_CNTL_GEN6 0x101008 +#define GFX_FLSH_CNTL _MMIO(0x2170) /* 915+ only */ +#define GFX_FLSH_CNTL_GEN6 _MMIO(0x101008) #define GFX_FLSH_CNTL_EN (1<<0) -#define ECOSKPD 0x021d0 +#define ECOSKPD _MMIO(0x21d0) #define ECO_GATING_CX_ONLY (1<<3) #define ECO_FLIP_DONE (1<<0) -#define CACHE_MODE_0_GEN7 0x7000 /* IVB+ */ +#define CACHE_MODE_0_GEN7 _MMIO(0x7000) /* IVB+ */ #define RC_OP_FLUSH_ENABLE (1<<0) #define HIZ_RAW_STALL_OPT_DISABLE (1<<2) -#define CACHE_MODE_1 0x7004 /* IVB+ */ +#define CACHE_MODE_1 _MMIO(0x7004) /* IVB+ */ #define PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1<<6) #define GEN8_4x4_STC_OPTIMIZATION_DISABLE (1<<6) #define GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE (1<<1) -#define GEN6_BLITTER_ECOSKPD 0x221d0 +#define GEN6_BLITTER_ECOSKPD _MMIO(0x221d0) #define GEN6_BLITTER_LOCK_SHIFT 16 #define GEN6_BLITTER_FBC_NOTIFY (1<<3) -#define GEN6_RC_SLEEP_PSMI_CONTROL 0x2050 +#define GEN6_RC_SLEEP_PSMI_CONTROL _MMIO(0x2050) #define GEN6_PSMI_SLEEP_MSG_DISABLE (1 << 0) #define GEN8_RC_SEMA_IDLE_MSG_DISABLE (1 << 12) #define GEN8_FF_DOP_CLOCK_GATE_DISABLE (1<<10) /* Fuse readout registers for GT */ -#define CHV_FUSE_GT (VLV_DISPLAY_BASE + 0x2168) +#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) #define CHV_FGT_DISABLE_SS0 (1 << 10) #define CHV_FGT_DISABLE_SS1 (1 << 11) #define CHV_FGT_EU_DIS_SS0_R0_SHIFT 16 @@ -1808,16 +1953,31 @@ enum skl_disp_power_wells { #define CHV_FGT_EU_DIS_SS1_R1_SHIFT 28 #define CHV_FGT_EU_DIS_SS1_R1_MASK (0xf << CHV_FGT_EU_DIS_SS1_R1_SHIFT) -#define GEN8_FUSE2 0x9120 +#define GEN8_FUSE2 _MMIO(0x9120) +#define GEN8_F2_SS_DIS_SHIFT 21 +#define GEN8_F2_SS_DIS_MASK (0x7 << GEN8_F2_SS_DIS_SHIFT) #define GEN8_F2_S_ENA_SHIFT 25 #define GEN8_F2_S_ENA_MASK (0x7 << GEN8_F2_S_ENA_SHIFT) #define GEN9_F2_SS_DIS_SHIFT 20 #define GEN9_F2_SS_DIS_MASK (0xf << GEN9_F2_SS_DIS_SHIFT) -#define GEN9_EU_DISABLE(slice) (0x9134 + (slice)*0x4) +#define GEN8_EU_DISABLE0 _MMIO(0x9134) +#define GEN8_EU_DIS0_S0_MASK 0xffffff +#define GEN8_EU_DIS0_S1_SHIFT 24 +#define GEN8_EU_DIS0_S1_MASK (0xff << GEN8_EU_DIS0_S1_SHIFT) -#define GEN6_BSD_SLEEP_PSMI_CONTROL 0x12050 +#define GEN8_EU_DISABLE1 _MMIO(0x9138) +#define GEN8_EU_DIS1_S1_MASK 0xffff +#define GEN8_EU_DIS1_S2_SHIFT 16 +#define GEN8_EU_DIS1_S2_MASK (0xffff << GEN8_EU_DIS1_S2_SHIFT) + +#define GEN8_EU_DISABLE2 _MMIO(0x913c) +#define GEN8_EU_DIS2_S2_MASK 0xff + +#define GEN9_EU_DISABLE(slice) _MMIO(0x9134 + (slice)*0x4) + +#define GEN6_BSD_SLEEP_PSMI_CONTROL _MMIO(0x12050) #define GEN6_BSD_SLEEP_MSG_DISABLE (1 << 0) #define GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2) #define GEN6_BSD_SLEEP_INDICATOR (1 << 3) @@ -1895,9 +2055,9 @@ enum skl_disp_power_wells { #define I915_ASLE_INTERRUPT (1<<0) #define I915_BSD_USER_INTERRUPT (1<<25) -#define GEN6_BSD_RNCID 0x12198 +#define GEN6_BSD_RNCID _MMIO(0x12198) -#define GEN7_FF_THREAD_MODE 0x20a0 +#define GEN7_FF_THREAD_MODE _MMIO(0x20a0) #define GEN7_FF_SCHED_MASK 0x0077070 #define GEN8_FF_DS_REF_CNT_FFME (1 << 19) #define GEN7_FF_TS_SCHED_HS1 (0x5<<16) @@ -1918,9 +2078,9 @@ enum skl_disp_power_wells { * Framebuffer compression (915+ only) */ -#define FBC_CFB_BASE 0x03200 /* 4k page aligned */ -#define FBC_LL_BASE 0x03204 /* 4k page aligned */ -#define FBC_CONTROL 0x03208 +#define FBC_CFB_BASE _MMIO(0x3200) /* 4k page aligned */ +#define FBC_LL_BASE _MMIO(0x3204) /* 4k page aligned */ +#define FBC_CONTROL _MMIO(0x3208) #define FBC_CTL_EN (1<<31) #define FBC_CTL_PERIODIC (1<<30) #define FBC_CTL_INTERVAL_SHIFT (16) @@ -1928,14 +2088,14 @@ enum skl_disp_power_wells { #define FBC_CTL_C3_IDLE (1<<13) #define FBC_CTL_STRIDE_SHIFT (5) #define FBC_CTL_FENCENO_SHIFT (0) -#define FBC_COMMAND 0x0320c +#define FBC_COMMAND _MMIO(0x320c) #define FBC_CMD_COMPRESS (1<<0) -#define FBC_STATUS 0x03210 +#define FBC_STATUS _MMIO(0x3210) #define FBC_STAT_COMPRESSING (1<<31) #define FBC_STAT_COMPRESSED (1<<30) #define FBC_STAT_MODIFIED (1<<29) #define FBC_STAT_CURRENT_LINE_SHIFT (0) -#define FBC_CONTROL2 0x03214 +#define FBC_CONTROL2 _MMIO(0x3214) #define FBC_CTL_FENCE_DBL (0<<4) #define FBC_CTL_IDLE_IMM (0<<2) #define FBC_CTL_IDLE_FULL (1<<2) @@ -1943,14 +2103,17 @@ enum skl_disp_power_wells { #define FBC_CTL_IDLE_DEBUG (3<<2) #define FBC_CTL_CPU_FENCE (1<<1) #define FBC_CTL_PLANE(plane) ((plane)<<0) -#define FBC_FENCE_OFF 0x03218 /* BSpec typo has 321Bh */ -#define FBC_TAG 0x03300 +#define FBC_FENCE_OFF _MMIO(0x3218) /* BSpec typo has 321Bh */ +#define FBC_TAG(i) _MMIO(0x3300 + (i) * 4) + +#define FBC_STATUS2 _MMIO(0x43214) +#define FBC_COMPRESSION_MASK 0x7ff #define FBC_LL_SIZE (1536) /* Framebuffer compression for GM45+ */ -#define DPFC_CB_BASE 0x3200 -#define DPFC_CONTROL 0x3208 +#define DPFC_CB_BASE _MMIO(0x3200) +#define DPFC_CONTROL _MMIO(0x3208) #define DPFC_CTL_EN (1<<31) #define DPFC_CTL_PLANE(plane) ((plane)<<30) #define IVB_DPFC_CTL_PLANE(plane) ((plane)<<29) @@ -1961,37 +2124,37 @@ enum skl_disp_power_wells { #define DPFC_CTL_LIMIT_1X (0<<6) #define DPFC_CTL_LIMIT_2X (1<<6) #define DPFC_CTL_LIMIT_4X (2<<6) -#define DPFC_RECOMP_CTL 0x320c +#define DPFC_RECOMP_CTL _MMIO(0x320c) #define DPFC_RECOMP_STALL_EN (1<<27) #define DPFC_RECOMP_STALL_WM_SHIFT (16) #define DPFC_RECOMP_STALL_WM_MASK (0x07ff0000) #define DPFC_RECOMP_TIMER_COUNT_SHIFT (0) #define DPFC_RECOMP_TIMER_COUNT_MASK (0x0000003f) -#define DPFC_STATUS 0x3210 +#define DPFC_STATUS _MMIO(0x3210) #define DPFC_INVAL_SEG_SHIFT (16) #define DPFC_INVAL_SEG_MASK (0x07ff0000) #define DPFC_COMP_SEG_SHIFT (0) #define DPFC_COMP_SEG_MASK (0x000003ff) -#define DPFC_STATUS2 0x3214 -#define DPFC_FENCE_YOFF 0x3218 -#define DPFC_CHICKEN 0x3224 +#define DPFC_STATUS2 _MMIO(0x3214) +#define DPFC_FENCE_YOFF _MMIO(0x3218) +#define DPFC_CHICKEN _MMIO(0x3224) #define DPFC_HT_MODIFY (1<<31) /* Framebuffer compression for Ironlake */ -#define ILK_DPFC_CB_BASE 0x43200 -#define ILK_DPFC_CONTROL 0x43208 +#define ILK_DPFC_CB_BASE _MMIO(0x43200) +#define ILK_DPFC_CONTROL _MMIO(0x43208) #define FBC_CTL_FALSE_COLOR (1<<10) /* The bit 28-8 is reserved */ #define DPFC_RESERVED (0x1FFFFF00) -#define ILK_DPFC_RECOMP_CTL 0x4320c -#define ILK_DPFC_STATUS 0x43210 -#define ILK_DPFC_FENCE_YOFF 0x43218 -#define ILK_DPFC_CHICKEN 0x43224 -#define ILK_FBC_RT_BASE 0x2128 +#define ILK_DPFC_RECOMP_CTL _MMIO(0x4320c) +#define ILK_DPFC_STATUS _MMIO(0x43210) +#define ILK_DPFC_FENCE_YOFF _MMIO(0x43218) +#define ILK_DPFC_CHICKEN _MMIO(0x43224) +#define ILK_FBC_RT_BASE _MMIO(0x2128) #define ILK_FBC_RT_VALID (1<<0) #define SNB_FBC_FRONT_BUFFER (1<<1) -#define ILK_DISPLAY_CHICKEN1 0x42000 +#define ILK_DISPLAY_CHICKEN1 _MMIO(0x42000) #define ILK_FBCQ_DIS (1<<22) #define ILK_PABSTRETCH_DIS (1<<21) @@ -2001,31 +2164,31 @@ enum skl_disp_power_wells { * * The following two registers are of type GTTMMADR */ -#define SNB_DPFC_CTL_SA 0x100100 +#define SNB_DPFC_CTL_SA _MMIO(0x100100) #define SNB_CPU_FENCE_ENABLE (1<<29) -#define DPFC_CPU_FENCE_OFFSET 0x100104 +#define DPFC_CPU_FENCE_OFFSET _MMIO(0x100104) /* Framebuffer compression for Ivybridge */ -#define IVB_FBC_RT_BASE 0x7020 +#define IVB_FBC_RT_BASE _MMIO(0x7020) -#define IPS_CTL 0x43408 +#define IPS_CTL _MMIO(0x43408) #define IPS_ENABLE (1 << 31) -#define MSG_FBC_REND_STATE 0x50380 +#define MSG_FBC_REND_STATE _MMIO(0x50380) #define FBC_REND_NUKE (1<<2) #define FBC_REND_CACHE_CLEAN (1<<1) /* * GPIO regs */ -#define GPIOA 0x5010 -#define GPIOB 0x5014 -#define GPIOC 0x5018 -#define GPIOD 0x501c -#define GPIOE 0x5020 -#define GPIOF 0x5024 -#define GPIOG 0x5028 -#define GPIOH 0x502c +#define GPIOA _MMIO(0x5010) +#define GPIOB _MMIO(0x5014) +#define GPIOC _MMIO(0x5018) +#define GPIOD _MMIO(0x501c) +#define GPIOE _MMIO(0x5020) +#define GPIOF _MMIO(0x5024) +#define GPIOG _MMIO(0x5028) +#define GPIOH _MMIO(0x502c) # define GPIO_CLOCK_DIR_MASK (1 << 0) # define GPIO_CLOCK_DIR_IN (0 << 1) # define GPIO_CLOCK_DIR_OUT (1 << 1) @@ -2041,7 +2204,7 @@ enum skl_disp_power_wells { # define GPIO_DATA_VAL_IN (1 << 12) # define GPIO_DATA_PULLUP_DISABLE (1 << 13) -#define GMBUS0 0x5100 /* clock/port select */ +#define GMBUS0 _MMIO(dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */ #define GMBUS_RATE_100KHZ (0<<8) #define GMBUS_RATE_50KHZ (1<<8) #define GMBUS_RATE_400KHZ (2<<8) /* reserved on Pineview */ @@ -2060,7 +2223,7 @@ enum skl_disp_power_wells { #define GMBUS_PIN_2_BXT 2 #define GMBUS_PIN_3_BXT 3 #define GMBUS_NUM_PINS 7 /* including 0 */ -#define GMBUS1 0x5104 /* command/status */ +#define GMBUS1 _MMIO(dev_priv->gpio_mmio_base + 0x5104) /* command/status */ #define GMBUS_SW_CLR_INT (1<<31) #define GMBUS_SW_RDY (1<<30) #define GMBUS_ENT (1<<29) /* enable timeout */ @@ -2074,7 +2237,7 @@ enum skl_disp_power_wells { #define GMBUS_SLAVE_ADDR_SHIFT 1 #define GMBUS_SLAVE_READ (1<<0) #define GMBUS_SLAVE_WRITE (0<<0) -#define GMBUS2 0x5108 /* status */ +#define GMBUS2 _MMIO(dev_priv->gpio_mmio_base + 0x5108) /* status */ #define GMBUS_INUSE (1<<15) #define GMBUS_HW_WAIT_PHASE (1<<14) #define GMBUS_STALL_TIMEOUT (1<<13) @@ -2082,14 +2245,14 @@ enum skl_disp_power_wells { #define GMBUS_HW_RDY (1<<11) #define GMBUS_SATOER (1<<10) #define GMBUS_ACTIVE (1<<9) -#define GMBUS3 0x510c /* data buffer bytes 3-0 */ -#define GMBUS4 0x5110 /* interrupt mask (Pineview+) */ +#define GMBUS3 _MMIO(dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */ +#define GMBUS4 _MMIO(dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */ #define GMBUS_SLAVE_TIMEOUT_EN (1<<4) #define GMBUS_NAK_EN (1<<3) #define GMBUS_IDLE_EN (1<<2) #define GMBUS_HW_WAIT_EN (1<<1) #define GMBUS_HW_RDY_EN (1<<0) -#define GMBUS5 0x5120 /* byte index */ +#define GMBUS5 _MMIO(dev_priv->gpio_mmio_base + 0x5120) /* byte index */ #define GMBUS_2BYTE_INDEX_EN (1<<31) /* @@ -2098,11 +2261,11 @@ enum skl_disp_power_wells { #define _DPLL_A (dev_priv->info.display_mmio_offset + 0x6014) #define _DPLL_B (dev_priv->info.display_mmio_offset + 0x6018) #define _CHV_DPLL_C (dev_priv->info.display_mmio_offset + 0x6030) -#define DPLL(pipe) _PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C) +#define DPLL(pipe) _MMIO_PIPE3((pipe), _DPLL_A, _DPLL_B, _CHV_DPLL_C) -#define VGA0 0x6000 -#define VGA1 0x6004 -#define VGA_PD 0x6010 +#define VGA0 _MMIO(0x6000) +#define VGA1 _MMIO(0x6004) +#define VGA_PD _MMIO(0x6010) #define VGA0_PD_P2_DIV_4 (1 << 7) #define VGA0_PD_P1_DIV_2 (1 << 5) #define VGA0_PD_P1_SHIFT 0 @@ -2116,7 +2279,7 @@ enum skl_disp_power_wells { #define DPLL_DVO_2X_MODE (1 << 30) #define DPLL_EXT_BUFFER_ENABLE_VLV (1 << 30) #define DPLL_SYNCLOCK_ENABLE (1 << 29) -#define DPLL_REFA_CLK_ENABLE_VLV (1 << 29) +#define DPLL_REF_CLK_ENABLE_VLV (1 << 29) #define DPLL_VGA_MODE_DIS (1 << 28) #define DPLLB_MODE_DAC_SERIAL (1 << 26) /* i915 */ #define DPLLB_MODE_LVDS (2 << 26) /* i915 */ @@ -2130,27 +2293,31 @@ enum skl_disp_power_wells { #define DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW 0x00ff8000 /* Pineview */ #define DPLL_LOCK_VLV (1<<15) #define DPLL_INTEGRATED_CRI_CLK_VLV (1<<14) -#define DPLL_INTEGRATED_CLOCK_VLV (1<<13) -#define DPLL_SSC_REF_CLOCK_CHV (1<<13) +#define DPLL_INTEGRATED_REF_CLK_VLV (1<<13) +#define DPLL_SSC_REF_CLK_CHV (1<<13) #define DPLL_PORTC_READY_MASK (0xf << 4) #define DPLL_PORTB_READY_MASK (0xf) #define DPLL_FPA01_P1_POST_DIV_MASK_I830 0x001f0000 /* Additional CHV pll/phy registers */ -#define DPIO_PHY_STATUS (VLV_DISPLAY_BASE + 0x6240) +#define DPIO_PHY_STATUS _MMIO(VLV_DISPLAY_BASE + 0x6240) #define DPLL_PORTD_READY_MASK (0xf) -#define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100) +#define DISPLAY_PHY_CONTROL _MMIO(VLV_DISPLAY_BASE + 0x60100) +#define PHY_CH_POWER_DOWN_OVRD_EN(phy, ch) (1 << (2*(phy)+(ch)+27)) #define PHY_LDO_DELAY_0NS 0x0 #define PHY_LDO_DELAY_200NS 0x1 #define PHY_LDO_DELAY_600NS 0x2 #define PHY_LDO_SEQ_DELAY(delay, phy) ((delay) << (2*(phy)+23)) +#define PHY_CH_POWER_DOWN_OVRD(mask, phy, ch) ((mask) << (8*(phy)+4*(ch)+11)) #define PHY_CH_SU_PSR 0x1 #define PHY_CH_DEEP_PSR 0x7 #define PHY_CH_POWER_MODE(mode, phy, ch) ((mode) << (6*(phy)+3*(ch)+2)) #define PHY_COM_LANE_RESET_DEASSERT(phy) (1 << (phy)) -#define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104) +#define DISPLAY_PHY_STATUS _MMIO(VLV_DISPLAY_BASE + 0x60104) #define PHY_POWERGOOD(phy) (((phy) == DPIO_PHY0) ? (1<<31) : (1<<30)) +#define PHY_STATUS_CMN_LDO(phy, ch) (1 << (6-(6*(phy)+3*(ch)))) +#define PHY_STATUS_SPLINE_LDO(phy, ch, spline) (1 << (8-(6*(phy)+3*(ch)+(spline)))) /* * The i830 generation, in LVDS mode, defines P1 as the bit number set within @@ -2193,7 +2360,7 @@ enum skl_disp_power_wells { #define _DPLL_A_MD (dev_priv->info.display_mmio_offset + 0x601c) #define _DPLL_B_MD (dev_priv->info.display_mmio_offset + 0x6020) #define _CHV_DPLL_C_MD (dev_priv->info.display_mmio_offset + 0x603c) -#define DPLL_MD(pipe) _PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD) +#define DPLL_MD(pipe) _MMIO_PIPE3((pipe), _DPLL_A_MD, _DPLL_B_MD, _CHV_DPLL_C_MD) /* * UDI pixel divider, controlling how many pixels are stuffed into a packet. @@ -2232,12 +2399,12 @@ enum skl_disp_power_wells { #define DPLL_MD_VGA_UDI_MULTIPLIER_MASK 0x0000003f #define DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT 0 -#define _FPA0 0x06040 -#define _FPA1 0x06044 -#define _FPB0 0x06048 -#define _FPB1 0x0604c -#define FP0(pipe) _PIPE(pipe, _FPA0, _FPB0) -#define FP1(pipe) _PIPE(pipe, _FPA1, _FPB1) +#define _FPA0 0x6040 +#define _FPA1 0x6044 +#define _FPB0 0x6048 +#define _FPB1 0x604c +#define FP0(pipe) _MMIO_PIPE(pipe, _FPA0, _FPB0) +#define FP1(pipe) _MMIO_PIPE(pipe, _FPA1, _FPB1) #define FP_N_DIV_MASK 0x003f0000 #define FP_N_PINEVIEW_DIV_MASK 0x00ff0000 #define FP_N_DIV_SHIFT 16 @@ -2246,7 +2413,7 @@ enum skl_disp_power_wells { #define FP_M2_DIV_MASK 0x0000003f #define FP_M2_PINEVIEW_DIV_MASK 0x000000ff #define FP_M2_DIV_SHIFT 0 -#define DPLL_TEST 0x606c +#define DPLL_TEST _MMIO(0x606c) #define DPLLB_TEST_SDVO_DIV_1 (0 << 22) #define DPLLB_TEST_SDVO_DIV_2 (1 << 22) #define DPLLB_TEST_SDVO_DIV_4 (2 << 22) @@ -2257,12 +2424,12 @@ enum skl_disp_power_wells { #define DPLLA_TEST_N_BYPASS (1 << 3) #define DPLLA_TEST_M_BYPASS (1 << 2) #define DPLLA_INPUT_BUFFER_ENABLE (1 << 0) -#define D_STATE 0x6104 +#define D_STATE _MMIO(0x6104) #define DSTATE_GFX_RESET_I830 (1<<6) #define DSTATE_PLL_D3_OFF (1<<3) #define DSTATE_GFX_CLOCK_GATING (1<<1) #define DSTATE_DOT_CLOCK_GATING (1<<0) -#define DSPCLK_GATE_D (dev_priv->info.display_mmio_offset + 0x6200) +#define DSPCLK_GATE_D _MMIO(dev_priv->info.display_mmio_offset + 0x6200) # define DPUNIT_B_CLOCK_GATE_DISABLE (1 << 30) /* 965 */ # define VSUNIT_CLOCK_GATE_DISABLE (1 << 29) /* 965 */ # define VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) /* 965 */ @@ -2301,7 +2468,7 @@ enum skl_disp_power_wells { # define ZVUNIT_CLOCK_GATE_DISABLE (1 << 0) /* 830 */ # define OVLUNIT_CLOCK_GATE_DISABLE (1 << 0) /* 845,865 */ -#define RENCLK_GATE_D1 0x6204 +#define RENCLK_GATE_D1 _MMIO(0x6204) # define BLITTER_CLOCK_GATE_DISABLE (1 << 13) /* 945GM only */ # define MPEG_CLOCK_GATE_DISABLE (1 << 12) /* 945GM only */ # define PC_FE_CLOCK_GATE_DISABLE (1 << 11) @@ -2365,35 +2532,35 @@ enum skl_disp_power_wells { # define I965_FT_CLOCK_GATE_DISABLE (1 << 1) # define I965_DM_CLOCK_GATE_DISABLE (1 << 0) -#define RENCLK_GATE_D2 0x6208 +#define RENCLK_GATE_D2 _MMIO(0x6208) #define VF_UNIT_CLOCK_GATE_DISABLE (1 << 9) #define GS_UNIT_CLOCK_GATE_DISABLE (1 << 7) #define CL_UNIT_CLOCK_GATE_DISABLE (1 << 6) -#define VDECCLK_GATE_D 0x620C /* g4x only */ +#define VDECCLK_GATE_D _MMIO(0x620C) /* g4x only */ #define VCP_UNIT_CLOCK_GATE_DISABLE (1 << 4) -#define RAMCLK_GATE_D 0x6210 /* CRL only */ -#define DEUC 0x6214 /* CRL only */ +#define RAMCLK_GATE_D _MMIO(0x6210) /* CRL only */ +#define DEUC _MMIO(0x6214) /* CRL only */ -#define FW_BLC_SELF_VLV (VLV_DISPLAY_BASE + 0x6500) +#define FW_BLC_SELF_VLV _MMIO(VLV_DISPLAY_BASE + 0x6500) #define FW_CSPWRDWNEN (1<<15) -#define MI_ARB_VLV (VLV_DISPLAY_BASE + 0x6504) +#define MI_ARB_VLV _MMIO(VLV_DISPLAY_BASE + 0x6504) -#define CZCLK_CDCLK_FREQ_RATIO (VLV_DISPLAY_BASE + 0x6508) +#define CZCLK_CDCLK_FREQ_RATIO _MMIO(VLV_DISPLAY_BASE + 0x6508) #define CDCLK_FREQ_SHIFT 4 #define CDCLK_FREQ_MASK (0x1f << CDCLK_FREQ_SHIFT) #define CZCLK_FREQ_MASK 0xf -#define GCI_CONTROL (VLV_DISPLAY_BASE + 0x650C) +#define GCI_CONTROL _MMIO(VLV_DISPLAY_BASE + 0x650C) #define PFI_CREDIT_63 (9 << 28) /* chv only */ #define PFI_CREDIT_31 (8 << 28) /* chv only */ #define PFI_CREDIT(x) (((x) - 8) << 28) /* 8-15 */ #define PFI_CREDIT_RESEND (1 << 27) #define VGA_FAST_MODE_DISABLE (1 << 14) -#define GMBUSFREQ_VLV (VLV_DISPLAY_BASE + 0x6510) +#define GMBUSFREQ_VLV _MMIO(VLV_DISPLAY_BASE + 0x6510) /* * Palette regs @@ -2401,8 +2568,8 @@ enum skl_disp_power_wells { #define PALETTE_A_OFFSET 0xa000 #define PALETTE_B_OFFSET 0xa800 #define CHV_PALETTE_C_OFFSET 0xc000 -#define PALETTE(pipe) (dev_priv->info.palette_offsets[pipe] + \ - dev_priv->info.display_mmio_offset) +#define PALETTE(pipe, i) _MMIO(dev_priv->info.palette_offsets[pipe] + \ + dev_priv->info.display_mmio_offset + (i) * 4) /* MCH MMIO space */ @@ -2420,32 +2587,37 @@ enum skl_disp_power_wells { #define MCHBAR_MIRROR_BASE_SNB 0x140000 +#define CTG_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x34) +#define ELK_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x48) +#define G4X_STOLEN_RESERVED_ADDR1_MASK (0xFFFF << 16) +#define G4X_STOLEN_RESERVED_ADDR2_MASK (0xFFF << 4) + /* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */ -#define DCLK (MCHBAR_MIRROR_BASE_SNB + 0x5e04) +#define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04) /* 915-945 and GM965 MCH register controlling DRAM channel access */ -#define DCC 0x10200 +#define DCC _MMIO(MCHBAR_MIRROR_BASE + 0x200) #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0) #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0) #define DCC_ADDRESSING_MODE_MASK (3 << 0) #define DCC_CHANNEL_XOR_DISABLE (1 << 10) #define DCC_CHANNEL_XOR_BIT_17 (1 << 9) -#define DCC2 0x10204 +#define DCC2 _MMIO(MCHBAR_MIRROR_BASE + 0x204) #define DCC2_MODIFIED_ENHANCED_DISABLE (1 << 20) /* Pineview MCH register contains DDR3 setting */ -#define CSHRDDR3CTL 0x101a8 +#define CSHRDDR3CTL _MMIO(MCHBAR_MIRROR_BASE + 0x1a8) #define CSHRDDR3CTL_DDR3 (1 << 2) /* 965 MCH register controlling DRAM channel configuration */ -#define C0DRB3 0x10206 -#define C1DRB3 0x10606 +#define C0DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x206) +#define C1DRB3 _MMIO(MCHBAR_MIRROR_BASE + 0x606) /* snb MCH registers for reading the DRAM channel configuration */ -#define MAD_DIMM_C0 (MCHBAR_MIRROR_BASE_SNB + 0x5004) -#define MAD_DIMM_C1 (MCHBAR_MIRROR_BASE_SNB + 0x5008) -#define MAD_DIMM_C2 (MCHBAR_MIRROR_BASE_SNB + 0x500C) +#define MAD_DIMM_C0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004) +#define MAD_DIMM_C1 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008) +#define MAD_DIMM_C2 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) #define MAD_DIMM_ECC_MASK (0x3 << 24) #define MAD_DIMM_ECC_OFF (0x0 << 24) #define MAD_DIMM_ECC_IO_ON_LOGIC_OFF (0x1 << 24) @@ -2465,14 +2637,14 @@ enum skl_disp_power_wells { #define MAD_DIMM_A_SIZE_MASK (0xff << MAD_DIMM_A_SIZE_SHIFT) /* snb MCH registers for priority tuning */ -#define MCH_SSKPD (MCHBAR_MIRROR_BASE_SNB + 0x5d10) +#define MCH_SSKPD _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10) #define MCH_SSKPD_WM0_MASK 0x3f #define MCH_SSKPD_WM0_VAL 0xc -#define MCH_SECP_NRG_STTS (MCHBAR_MIRROR_BASE_SNB + 0x592c) +#define MCH_SECP_NRG_STTS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x592c) /* Clocking configuration register */ -#define CLKCFG 0x10c00 +#define CLKCFG _MMIO(MCHBAR_MIRROR_BASE + 0xc00) #define CLKCFG_FSB_400 (5 << 0) /* hrawclk 100 */ #define CLKCFG_FSB_533 (1 << 0) /* hrawclk 133 */ #define CLKCFG_FSB_667 (3 << 0) /* hrawclk 166 */ @@ -2488,23 +2660,26 @@ enum skl_disp_power_wells { #define CLKCFG_MEM_800 (3 << 4) #define CLKCFG_MEM_MASK (7 << 4) -#define TSC1 0x11001 +#define HPLLVCO _MMIO(MCHBAR_MIRROR_BASE + 0xc38) +#define HPLLVCO_MOBILE _MMIO(MCHBAR_MIRROR_BASE + 0xc0f) + +#define TSC1 _MMIO(0x11001) #define TSE (1<<0) -#define TR1 0x11006 -#define TSFS 0x11020 +#define TR1 _MMIO(0x11006) +#define TSFS _MMIO(0x11020) #define TSFS_SLOPE_MASK 0x0000ff00 #define TSFS_SLOPE_SHIFT 8 #define TSFS_INTR_MASK 0x000000ff -#define CRSTANDVID 0x11100 -#define PXVFREQ_BASE 0x11110 /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */ +#define CRSTANDVID _MMIO(0x11100) +#define PXVFREQ(fstart) _MMIO(0x11110 + (fstart) * 4) /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */ #define PXVFREQ_PX_MASK 0x7f000000 #define PXVFREQ_PX_SHIFT 24 -#define VIDFREQ_BASE 0x11110 -#define VIDFREQ1 0x11110 /* VIDFREQ1-4 (0x1111c) (Cantiga) */ -#define VIDFREQ2 0x11114 -#define VIDFREQ3 0x11118 -#define VIDFREQ4 0x1111c +#define VIDFREQ_BASE _MMIO(0x11110) +#define VIDFREQ1 _MMIO(0x11110) /* VIDFREQ1-4 (0x1111c) (Cantiga) */ +#define VIDFREQ2 _MMIO(0x11114) +#define VIDFREQ3 _MMIO(0x11118) +#define VIDFREQ4 _MMIO(0x1111c) #define VIDFREQ_P0_MASK 0x1f000000 #define VIDFREQ_P0_SHIFT 24 #define VIDFREQ_P0_CSCLK_MASK 0x00f00000 @@ -2516,8 +2691,8 @@ enum skl_disp_power_wells { #define VIDFREQ_P1_CSCLK_MASK 0x000000f0 #define VIDFREQ_P1_CSCLK_SHIFT 4 #define VIDFREQ_P1_CRCLK_MASK 0x0000000f -#define INTTOEXT_BASE_ILK 0x11300 -#define INTTOEXT_BASE 0x11120 /* INTTOEXT1-8 (0x1113c) */ +#define INTTOEXT_BASE_ILK _MMIO(0x11300) +#define INTTOEXT_BASE _MMIO(0x11120) /* INTTOEXT1-8 (0x1113c) */ #define INTTOEXT_MAP3_SHIFT 24 #define INTTOEXT_MAP3_MASK (0x1f << INTTOEXT_MAP3_SHIFT) #define INTTOEXT_MAP2_SHIFT 16 @@ -2526,7 +2701,7 @@ enum skl_disp_power_wells { #define INTTOEXT_MAP1_MASK (0x1f << INTTOEXT_MAP1_SHIFT) #define INTTOEXT_MAP0_SHIFT 0 #define INTTOEXT_MAP0_MASK (0x1f << INTTOEXT_MAP0_SHIFT) -#define MEMSWCTL 0x11170 /* Ironlake only */ +#define MEMSWCTL _MMIO(0x11170) /* Ironlake only */ #define MEMCTL_CMD_MASK 0xe000 #define MEMCTL_CMD_SHIFT 13 #define MEMCTL_CMD_RCLK_OFF 0 @@ -2541,8 +2716,8 @@ enum skl_disp_power_wells { #define MEMCTL_FREQ_SHIFT 8 #define MEMCTL_SFCAVM (1<<7) #define MEMCTL_TGT_VID_MASK 0x007f -#define MEMIHYST 0x1117c -#define MEMINTREN 0x11180 /* 16 bits */ +#define MEMIHYST _MMIO(0x1117c) +#define MEMINTREN _MMIO(0x11180) /* 16 bits */ #define MEMINT_RSEXIT_EN (1<<8) #define MEMINT_CX_SUPR_EN (1<<7) #define MEMINT_CONT_BUSY_EN (1<<6) @@ -2552,7 +2727,7 @@ enum skl_disp_power_wells { #define MEMINT_UP_EVAL_EN (1<<2) #define MEMINT_DOWN_EVAL_EN (1<<1) #define MEMINT_SW_CMD_EN (1<<0) -#define MEMINTRSTR 0x11182 /* 16 bits */ +#define MEMINTRSTR _MMIO(0x11182) /* 16 bits */ #define MEM_RSEXIT_MASK 0xc000 #define MEM_RSEXIT_SHIFT 14 #define MEM_CONT_BUSY_MASK 0x3000 @@ -2572,7 +2747,7 @@ enum skl_disp_power_wells { #define MEM_INT_STEER_CMR 1 #define MEM_INT_STEER_SMI 2 #define MEM_INT_STEER_SCI 3 -#define MEMINTRSTS 0x11184 +#define MEMINTRSTS _MMIO(0x11184) #define MEMINT_RSEXIT (1<<7) #define MEMINT_CONT_BUSY (1<<6) #define MEMINT_AVG_BUSY (1<<5) @@ -2581,7 +2756,7 @@ enum skl_disp_power_wells { #define MEMINT_UP_EVAL (1<<2) #define MEMINT_DOWN_EVAL (1<<1) #define MEMINT_SW_CMD (1<<0) -#define MEMMODECTL 0x11190 +#define MEMMODECTL _MMIO(0x11190) #define MEMMODE_BOOST_EN (1<<31) #define MEMMODE_BOOST_FREQ_MASK 0x0f000000 /* jitter for boost, 0-15 */ #define MEMMODE_BOOST_FREQ_SHIFT 24 @@ -2598,8 +2773,8 @@ enum skl_disp_power_wells { #define MEMMODE_FMAX_MASK 0x000000f0 /* max jitter, 0-15 */ #define MEMMODE_FMAX_SHIFT 4 #define MEMMODE_FMIN_MASK 0x0000000f /* min jitter, 0-15 */ -#define RCBMAXAVG 0x1119c -#define MEMSWCTL2 0x1119e /* Cantiga only */ +#define RCBMAXAVG _MMIO(0x1119c) +#define MEMSWCTL2 _MMIO(0x1119e) /* Cantiga only */ #define SWMEMCMD_RENDER_OFF (0 << 13) #define SWMEMCMD_RENDER_ON (1 << 13) #define SWMEMCMD_SWFREQ (2 << 13) @@ -2611,11 +2786,11 @@ enum skl_disp_power_wells { #define SWFREQ_MASK 0x0380 /* P0-7 */ #define SWFREQ_SHIFT 7 #define TARVID_MASK 0x001f -#define MEMSTAT_CTG 0x111a0 -#define RCBMINAVG 0x111a0 -#define RCUPEI 0x111b0 -#define RCDNEI 0x111b4 -#define RSTDBYCTL 0x111b8 +#define MEMSTAT_CTG _MMIO(0x111a0) +#define RCBMINAVG _MMIO(0x111a0) +#define RCUPEI _MMIO(0x111b0) +#define RCDNEI _MMIO(0x111b4) +#define RSTDBYCTL _MMIO(0x111b8) #define RS1EN (1<<31) #define RS2EN (1<<30) #define RS3EN (1<<29) @@ -2659,10 +2834,10 @@ enum skl_disp_power_wells { #define RS_CSTATE_C367_RS2 (3<<4) #define REDSAVES (1<<3) /* no context save if was idle during rs0 */ #define REDRESTORES (1<<2) /* no restore if was idle during rs0 */ -#define VIDCTL 0x111c0 -#define VIDSTS 0x111c8 -#define VIDSTART 0x111cc /* 8 bits */ -#define MEMSTAT_ILK 0x111f8 +#define VIDCTL _MMIO(0x111c0) +#define VIDSTS _MMIO(0x111c8) +#define VIDSTART _MMIO(0x111cc) /* 8 bits */ +#define MEMSTAT_ILK _MMIO(0x111f8) #define MEMSTAT_VID_MASK 0x7f00 #define MEMSTAT_VID_SHIFT 8 #define MEMSTAT_PSTATE_MASK 0x00f8 @@ -2673,64 +2848,69 @@ enum skl_disp_power_wells { #define MEMSTAT_SRC_CTL_TRB 1 #define MEMSTAT_SRC_CTL_THM 2 #define MEMSTAT_SRC_CTL_STDBY 3 -#define RCPREVBSYTUPAVG 0x113b8 -#define RCPREVBSYTDNAVG 0x113bc -#define PMMISC 0x11214 +#define RCPREVBSYTUPAVG _MMIO(0x113b8) +#define RCPREVBSYTDNAVG _MMIO(0x113bc) +#define PMMISC _MMIO(0x11214) #define MCPPCE_EN (1<<0) /* enable PM_MSG from PCH->MPC */ -#define SDEW 0x1124c -#define CSIEW0 0x11250 -#define CSIEW1 0x11254 -#define CSIEW2 0x11258 -#define PEW 0x1125c -#define DEW 0x11270 -#define MCHAFE 0x112c0 -#define CSIEC 0x112e0 -#define DMIEC 0x112e4 -#define DDREC 0x112e8 -#define PEG0EC 0x112ec -#define PEG1EC 0x112f0 -#define GFXEC 0x112f4 -#define RPPREVBSYTUPAVG 0x113b8 -#define RPPREVBSYTDNAVG 0x113bc -#define ECR 0x11600 +#define SDEW _MMIO(0x1124c) +#define CSIEW0 _MMIO(0x11250) +#define CSIEW1 _MMIO(0x11254) +#define CSIEW2 _MMIO(0x11258) +#define PEW(i) _MMIO(0x1125c + (i) * 4) /* 5 registers */ +#define DEW(i) _MMIO(0x11270 + (i) * 4) /* 3 registers */ +#define MCHAFE _MMIO(0x112c0) +#define CSIEC _MMIO(0x112e0) +#define DMIEC _MMIO(0x112e4) +#define DDREC _MMIO(0x112e8) +#define PEG0EC _MMIO(0x112ec) +#define PEG1EC _MMIO(0x112f0) +#define GFXEC _MMIO(0x112f4) +#define RPPREVBSYTUPAVG _MMIO(0x113b8) +#define RPPREVBSYTDNAVG _MMIO(0x113bc) +#define ECR _MMIO(0x11600) #define ECR_GPFE (1<<31) #define ECR_IMONE (1<<30) #define ECR_CAP_MASK 0x0000001f /* Event range, 0-31 */ -#define OGW0 0x11608 -#define OGW1 0x1160c -#define EG0 0x11610 -#define EG1 0x11614 -#define EG2 0x11618 -#define EG3 0x1161c -#define EG4 0x11620 -#define EG5 0x11624 -#define EG6 0x11628 -#define EG7 0x1162c -#define PXW 0x11664 -#define PXWL 0x11680 -#define LCFUSE02 0x116c0 +#define OGW0 _MMIO(0x11608) +#define OGW1 _MMIO(0x1160c) +#define EG0 _MMIO(0x11610) +#define EG1 _MMIO(0x11614) +#define EG2 _MMIO(0x11618) +#define EG3 _MMIO(0x1161c) +#define EG4 _MMIO(0x11620) +#define EG5 _MMIO(0x11624) +#define EG6 _MMIO(0x11628) +#define EG7 _MMIO(0x1162c) +#define PXW(i) _MMIO(0x11664 + (i) * 4) /* 4 registers */ +#define PXWL(i) _MMIO(0x11680 + (i) * 8) /* 8 registers */ +#define LCFUSE02 _MMIO(0x116c0) #define LCFUSE_HIV_MASK 0x000000ff -#define CSIPLL0 0x12c10 -#define DDRMPLL1 0X12c20 -#define PEG_BAND_GAP_DATA 0x14d68 +#define CSIPLL0 _MMIO(0x12c10) +#define DDRMPLL1 _MMIO(0X12c20) +#define PEG_BAND_GAP_DATA _MMIO(0x14d68) -#define GEN6_GT_THREAD_STATUS_REG 0x13805c +#define GEN6_GT_THREAD_STATUS_REG _MMIO(0x13805c) #define GEN6_GT_THREAD_STATUS_CORE_MASK 0x7 -#define GEN6_GT_PERF_STATUS (MCHBAR_MIRROR_BASE_SNB + 0x5948) -#define GEN6_RP_STATE_LIMITS (MCHBAR_MIRROR_BASE_SNB + 0x5994) -#define GEN6_RP_STATE_CAP (MCHBAR_MIRROR_BASE_SNB + 0x5998) +#define GEN6_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948) +#define BXT_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070) +#define GEN6_RP_STATE_LIMITS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994) +#define GEN6_RP_STATE_CAP _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998) +#define BXT_RP_STATE_CAP _MMIO(0x138170) #define INTERVAL_1_28_US(us) (((us) * 100) >> 7) #define INTERVAL_1_33_US(us) (((us) * 3) >> 2) +#define INTERVAL_0_833_US(us) (((us) * 6) / 5) #define GT_INTERVAL_FROM_US(dev_priv, us) (IS_GEN9(dev_priv) ? \ - INTERVAL_1_33_US(us) : \ + (IS_BROXTON(dev_priv) ? \ + INTERVAL_0_833_US(us) : \ + INTERVAL_1_33_US(us)) : \ INTERVAL_1_28_US(us)) /* * Logical Context regs */ -#define CCID 0x2180 +#define CCID _MMIO(0x2180) #define CCID_EN (1<<0) /* * Notes on SNB/IVB/VLV context size: @@ -2745,51 +2925,59 @@ enum skl_disp_power_wells { * - GT1 size just indicates how much of render context * doesn't need saving on GT1 */ -#define CXT_SIZE 0x21a0 -#define GEN6_CXT_POWER_SIZE(cxt_reg) ((cxt_reg >> 24) & 0x3f) -#define GEN6_CXT_RING_SIZE(cxt_reg) ((cxt_reg >> 18) & 0x3f) -#define GEN6_CXT_RENDER_SIZE(cxt_reg) ((cxt_reg >> 12) & 0x3f) -#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) ((cxt_reg >> 6) & 0x3f) -#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) ((cxt_reg >> 0) & 0x3f) +#define CXT_SIZE _MMIO(0x21a0) +#define GEN6_CXT_POWER_SIZE(cxt_reg) (((cxt_reg) >> 24) & 0x3f) +#define GEN6_CXT_RING_SIZE(cxt_reg) (((cxt_reg) >> 18) & 0x3f) +#define GEN6_CXT_RENDER_SIZE(cxt_reg) (((cxt_reg) >> 12) & 0x3f) +#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) (((cxt_reg) >> 6) & 0x3f) +#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) (((cxt_reg) >> 0) & 0x3f) #define GEN6_CXT_TOTAL_SIZE(cxt_reg) (GEN6_CXT_RING_SIZE(cxt_reg) + \ GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \ GEN6_CXT_PIPELINE_SIZE(cxt_reg)) -#define GEN7_CXT_SIZE 0x21a8 -#define GEN7_CXT_POWER_SIZE(ctx_reg) ((ctx_reg >> 25) & 0x7f) -#define GEN7_CXT_RING_SIZE(ctx_reg) ((ctx_reg >> 22) & 0x7) -#define GEN7_CXT_RENDER_SIZE(ctx_reg) ((ctx_reg >> 16) & 0x3f) -#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) ((ctx_reg >> 9) & 0x7f) -#define GEN7_CXT_GT1_SIZE(ctx_reg) ((ctx_reg >> 6) & 0x7) -#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) ((ctx_reg >> 0) & 0x3f) +#define GEN7_CXT_SIZE _MMIO(0x21a8) +#define GEN7_CXT_POWER_SIZE(ctx_reg) (((ctx_reg) >> 25) & 0x7f) +#define GEN7_CXT_RING_SIZE(ctx_reg) (((ctx_reg) >> 22) & 0x7) +#define GEN7_CXT_RENDER_SIZE(ctx_reg) (((ctx_reg) >> 16) & 0x3f) +#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) (((ctx_reg) >> 9) & 0x7f) +#define GEN7_CXT_GT1_SIZE(ctx_reg) (((ctx_reg) >> 6) & 0x7) +#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) (((ctx_reg) >> 0) & 0x3f) #define GEN7_CXT_TOTAL_SIZE(ctx_reg) (GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \ GEN7_CXT_VFSTATE_SIZE(ctx_reg)) /* Haswell does have the CXT_SIZE register however it does not appear to be * valid. Now, docs explain in dwords what is in the context object. The full * size is 70720 bytes, however, the power context and execlist context will * never be saved (power context is stored elsewhere, and execlists don't work - * on HSW) - so the final size is 66944 bytes, which rounds to 17 pages. + * on HSW) - so the final size, including the extra state required for the + * Resource Streamer, is 66944 bytes, which rounds to 17 pages. */ #define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE) /* Same as Haswell, but 72064 bytes now. */ #define GEN8_CXT_TOTAL_SIZE (18 * PAGE_SIZE) -#define CHV_CLK_CTL1 0x101100 -#define VLV_CLK_CTL2 0x101104 +#define CHV_CLK_CTL1 _MMIO(0x101100) +#define VLV_CLK_CTL2 _MMIO(0x101104) #define CLK_CTL2_CZCOUNT_30NS_SHIFT 28 /* * Overlay regs */ -#define OVADD 0x30000 -#define DOVSTA 0x30008 +#define OVADD _MMIO(0x30000) +#define DOVSTA _MMIO(0x30008) #define OC_BUF (0x3<<20) -#define OGAMC5 0x30010 -#define OGAMC4 0x30014 -#define OGAMC3 0x30018 -#define OGAMC2 0x3001c -#define OGAMC1 0x30020 -#define OGAMC0 0x30024 +#define OGAMC5 _MMIO(0x30010) +#define OGAMC4 _MMIO(0x30014) +#define OGAMC3 _MMIO(0x30018) +#define OGAMC2 _MMIO(0x3001c) +#define OGAMC1 _MMIO(0x30020) +#define OGAMC0 _MMIO(0x30024) + +/* + * GEN9 clock gating regs + */ +#define GEN9_CLKGATE_DIS_0 _MMIO(0x46530) +#define PWM2_GATING_DIS (1 << 14) +#define PWM1_GATING_DIS (1 << 13) /* * Display engine regs @@ -2849,28 +3037,18 @@ enum skl_disp_power_wells { #define _PIPE_CRC_RES_4_B_IVB 0x61070 #define _PIPE_CRC_RES_5_B_IVB 0x61074 -#define PIPE_CRC_CTL(pipe) _TRANSCODER2(pipe, _PIPE_CRC_CTL_A) -#define PIPE_CRC_RES_1_IVB(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_1_A_IVB) -#define PIPE_CRC_RES_2_IVB(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_2_A_IVB) -#define PIPE_CRC_RES_3_IVB(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_3_A_IVB) -#define PIPE_CRC_RES_4_IVB(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_4_A_IVB) -#define PIPE_CRC_RES_5_IVB(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_5_A_IVB) - -#define PIPE_CRC_RES_RED(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_RED_A) -#define PIPE_CRC_RES_GREEN(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_GREEN_A) -#define PIPE_CRC_RES_BLUE(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_BLUE_A) -#define PIPE_CRC_RES_RES1_I915(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_RES1_A_I915) -#define PIPE_CRC_RES_RES2_G4X(pipe) \ - _TRANSCODER2(pipe, _PIPE_CRC_RES_RES2_A_G4X) +#define PIPE_CRC_CTL(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_CTL_A) +#define PIPE_CRC_RES_1_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_1_A_IVB) +#define PIPE_CRC_RES_2_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_2_A_IVB) +#define PIPE_CRC_RES_3_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_3_A_IVB) +#define PIPE_CRC_RES_4_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_4_A_IVB) +#define PIPE_CRC_RES_5_IVB(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_5_A_IVB) + +#define PIPE_CRC_RES_RED(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_RED_A) +#define PIPE_CRC_RES_GREEN(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_GREEN_A) +#define PIPE_CRC_RES_BLUE(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_BLUE_A) +#define PIPE_CRC_RES_RES1_I915(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES1_A_I915) +#define PIPE_CRC_RES_RES2_G4X(pipe) _MMIO_TRANS2(pipe, _PIPE_CRC_RES_RES2_A_G4X) /* Pipe A timing regs */ #define _HTOTAL_A 0x60000 @@ -2902,20 +3080,20 @@ enum skl_disp_power_wells { #define CHV_TRANSCODER_C_OFFSET 0x63000 #define TRANSCODER_EDP_OFFSET 0x6f000 -#define _TRANSCODER2(pipe, reg) (dev_priv->info.trans_offsets[(pipe)] - \ +#define _MMIO_TRANS2(pipe, reg) _MMIO(dev_priv->info.trans_offsets[(pipe)] - \ dev_priv->info.trans_offsets[TRANSCODER_A] + (reg) + \ dev_priv->info.display_mmio_offset) -#define HTOTAL(trans) _TRANSCODER2(trans, _HTOTAL_A) -#define HBLANK(trans) _TRANSCODER2(trans, _HBLANK_A) -#define HSYNC(trans) _TRANSCODER2(trans, _HSYNC_A) -#define VTOTAL(trans) _TRANSCODER2(trans, _VTOTAL_A) -#define VBLANK(trans) _TRANSCODER2(trans, _VBLANK_A) -#define VSYNC(trans) _TRANSCODER2(trans, _VSYNC_A) -#define BCLRPAT(trans) _TRANSCODER2(trans, _BCLRPAT_A) -#define VSYNCSHIFT(trans) _TRANSCODER2(trans, _VSYNCSHIFT_A) -#define PIPESRC(trans) _TRANSCODER2(trans, _PIPEASRC) -#define PIPE_MULT(trans) _TRANSCODER2(trans, _PIPE_MULT_A) +#define HTOTAL(trans) _MMIO_TRANS2(trans, _HTOTAL_A) +#define HBLANK(trans) _MMIO_TRANS2(trans, _HBLANK_A) +#define HSYNC(trans) _MMIO_TRANS2(trans, _HSYNC_A) +#define VTOTAL(trans) _MMIO_TRANS2(trans, _VTOTAL_A) +#define VBLANK(trans) _MMIO_TRANS2(trans, _VBLANK_A) +#define VSYNC(trans) _MMIO_TRANS2(trans, _VSYNC_A) +#define BCLRPAT(trans) _MMIO_TRANS2(trans, _BCLRPAT_A) +#define VSYNCSHIFT(trans) _MMIO_TRANS2(trans, _VSYNCSHIFT_A) +#define PIPESRC(trans) _MMIO_TRANS2(trans, _PIPEASRC) +#define PIPE_MULT(trans) _MMIO_TRANS2(trans, _PIPE_MULT_A) /* VLV eDP PSR registers */ #define _PSRCTLA (VLV_DISPLAY_BASE + 0x60090) @@ -2931,14 +3109,14 @@ enum skl_disp_power_wells { #define VLV_EDP_PSR_DBL_FRAME (1<<10) #define VLV_EDP_PSR_FRAME_COUNT_MASK (0xff<<16) #define VLV_EDP_PSR_IDLE_FRAME_SHIFT 16 -#define VLV_PSRCTL(pipe) _PIPE(pipe, _PSRCTLA, _PSRCTLB) +#define VLV_PSRCTL(pipe) _MMIO_PIPE(pipe, _PSRCTLA, _PSRCTLB) #define _VSCSDPA (VLV_DISPLAY_BASE + 0x600a0) #define _VSCSDPB (VLV_DISPLAY_BASE + 0x610a0) #define VLV_EDP_PSR_SDP_FREQ_MASK (3<<30) #define VLV_EDP_PSR_SDP_FREQ_ONCE (1<<31) #define VLV_EDP_PSR_SDP_FREQ_EVFRAME (1<<30) -#define VLV_VSCSDP(pipe) _PIPE(pipe, _VSCSDPA, _VSCSDPB) +#define VLV_VSCSDP(pipe) _MMIO_PIPE(pipe, _VSCSDPA, _VSCSDPB) #define _PSRSTATA (VLV_DISPLAY_BASE + 0x60094) #define _PSRSTATB (VLV_DISPLAY_BASE + 0x61094) @@ -2951,11 +3129,12 @@ enum skl_disp_power_wells { #define VLV_EDP_PSR_ACTIVE_SF_UPDATE (4<<0) #define VLV_EDP_PSR_EXIT (5<<0) #define VLV_EDP_PSR_IN_TRANS (1<<7) -#define VLV_PSRSTAT(pipe) _PIPE(pipe, _PSRSTATA, _PSRSTATB) +#define VLV_PSRSTAT(pipe) _MMIO_PIPE(pipe, _PSRSTATA, _PSRSTATB) /* HSW+ eDP PSR registers */ -#define EDP_PSR_BASE(dev) (IS_HASWELL(dev) ? 0x64800 : 0x6f800) -#define EDP_PSR_CTL(dev) (EDP_PSR_BASE(dev) + 0) +#define HSW_EDP_PSR_BASE 0x64800 +#define BDW_EDP_PSR_BASE 0x6f800 +#define EDP_PSR_CTL _MMIO(dev_priv->psr_mmio_base + 0) #define EDP_PSR_ENABLE (1<<31) #define BDW_PSR_SINGLE_FRAME (1<<30) #define EDP_PSR_LINK_STANDBY (1<<27) @@ -2978,14 +3157,10 @@ enum skl_disp_power_wells { #define EDP_PSR_TP1_TIME_0us (3<<4) #define EDP_PSR_IDLE_FRAME_SHIFT 0 -#define EDP_PSR_AUX_CTL(dev) (EDP_PSR_BASE(dev) + 0x10) -#define EDP_PSR_AUX_DATA1(dev) (EDP_PSR_BASE(dev) + 0x14) -#define EDP_PSR_AUX_DATA2(dev) (EDP_PSR_BASE(dev) + 0x18) -#define EDP_PSR_AUX_DATA3(dev) (EDP_PSR_BASE(dev) + 0x1c) -#define EDP_PSR_AUX_DATA4(dev) (EDP_PSR_BASE(dev) + 0x20) -#define EDP_PSR_AUX_DATA5(dev) (EDP_PSR_BASE(dev) + 0x24) +#define EDP_PSR_AUX_CTL _MMIO(dev_priv->psr_mmio_base + 0x10) +#define EDP_PSR_AUX_DATA(i) _MMIO(dev_priv->psr_mmio_base + 0x14 + (i) * 4) /* 5 registers */ -#define EDP_PSR_STATUS_CTL(dev) (EDP_PSR_BASE(dev) + 0x40) +#define EDP_PSR_STATUS_CTL _MMIO(dev_priv->psr_mmio_base + 0x40) #define EDP_PSR_STATUS_STATE_MASK (7<<29) #define EDP_PSR_STATUS_STATE_IDLE (0<<29) #define EDP_PSR_STATUS_STATE_SRDONACK (1<<29) @@ -3009,15 +3184,15 @@ enum skl_disp_power_wells { #define EDP_PSR_STATUS_SENDING_TP1 (1<<4) #define EDP_PSR_STATUS_IDLE_MASK 0xf -#define EDP_PSR_PERF_CNT(dev) (EDP_PSR_BASE(dev) + 0x44) +#define EDP_PSR_PERF_CNT _MMIO(dev_priv->psr_mmio_base + 0x44) #define EDP_PSR_PERF_CNT_MASK 0xffffff -#define EDP_PSR_DEBUG_CTL(dev) (EDP_PSR_BASE(dev) + 0x60) +#define EDP_PSR_DEBUG_CTL _MMIO(dev_priv->psr_mmio_base + 0x60) #define EDP_PSR_DEBUG_MASK_LPSP (1<<27) #define EDP_PSR_DEBUG_MASK_MEMUP (1<<26) #define EDP_PSR_DEBUG_MASK_HPD (1<<25) -#define EDP_PSR2_CTL 0x6f900 +#define EDP_PSR2_CTL _MMIO(0x6f900) #define EDP_PSR2_ENABLE (1<<31) #define EDP_SU_TRACK_ENABLE (1<<30) #define EDP_MAX_SU_DISABLE_TIME(t) ((t)<<20) @@ -3032,9 +3207,9 @@ enum skl_disp_power_wells { #define EDP_PSR2_IDLE_MASK 0xf /* VGA port control */ -#define ADPA 0x61100 -#define PCH_ADPA 0xe1100 -#define VLV_ADPA (VLV_DISPLAY_BASE + ADPA) +#define ADPA _MMIO(0x61100) +#define PCH_ADPA _MMIO(0xe1100) +#define VLV_ADPA _MMIO(VLV_DISPLAY_BASE + 0x61100) #define ADPA_DAC_ENABLE (1<<31) #define ADPA_DAC_DISABLE 0 @@ -3080,7 +3255,7 @@ enum skl_disp_power_wells { /* Hotplug control (945+ only) */ -#define PORT_HOTPLUG_EN (dev_priv->info.display_mmio_offset + 0x61110) +#define PORT_HOTPLUG_EN _MMIO(dev_priv->info.display_mmio_offset + 0x61110) #define PORTB_HOTPLUG_INT_EN (1 << 29) #define PORTC_HOTPLUG_INT_EN (1 << 28) #define PORTD_HOTPLUG_INT_EN (1 << 27) @@ -3110,7 +3285,7 @@ enum skl_disp_power_wells { #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV (0 << 2) #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV (1 << 2) -#define PORT_HOTPLUG_STAT (dev_priv->info.display_mmio_offset + 0x61114) +#define PORT_HOTPLUG_STAT _MMIO(dev_priv->info.display_mmio_offset + 0x61114) /* * HDMI/DP bits are gen4+ * @@ -3175,19 +3350,23 @@ enum skl_disp_power_wells { /* SDVO and HDMI port control. * The same register may be used for SDVO or HDMI */ -#define GEN3_SDVOB 0x61140 -#define GEN3_SDVOC 0x61160 +#define _GEN3_SDVOB 0x61140 +#define _GEN3_SDVOC 0x61160 +#define GEN3_SDVOB _MMIO(_GEN3_SDVOB) +#define GEN3_SDVOC _MMIO(_GEN3_SDVOC) #define GEN4_HDMIB GEN3_SDVOB #define GEN4_HDMIC GEN3_SDVOC -#define CHV_HDMID 0x6116C -#define PCH_SDVOB 0xe1140 +#define VLV_HDMIB _MMIO(VLV_DISPLAY_BASE + 0x61140) +#define VLV_HDMIC _MMIO(VLV_DISPLAY_BASE + 0x61160) +#define CHV_HDMID _MMIO(VLV_DISPLAY_BASE + 0x6116C) +#define PCH_SDVOB _MMIO(0xe1140) #define PCH_HDMIB PCH_SDVOB -#define PCH_HDMIC 0xe1150 -#define PCH_HDMID 0xe1160 +#define PCH_HDMIC _MMIO(0xe1150) +#define PCH_HDMID _MMIO(0xe1160) -#define PORT_DFT_I9XX 0x61150 +#define PORT_DFT_I9XX _MMIO(0x61150) #define DC_BALANCE_RESET (1 << 25) -#define PORT_DFT2_G4X (dev_priv->info.display_mmio_offset + 0x61154) +#define PORT_DFT2_G4X _MMIO(dev_priv->info.display_mmio_offset + 0x61154) #define DC_BALANCE_RESET_VLV (1 << 31) #define PIPE_SCRAMBLE_RESET_MASK ((1 << 14) | (0x3 << 0)) #define PIPE_C_SCRAMBLE_RESET (1 << 14) /* chv */ @@ -3247,9 +3426,12 @@ enum skl_disp_power_wells { /* DVO port control */ -#define DVOA 0x61120 -#define DVOB 0x61140 -#define DVOC 0x61160 +#define _DVOA 0x61120 +#define DVOA _MMIO(_DVOA) +#define _DVOB 0x61140 +#define DVOB _MMIO(_DVOB) +#define _DVOC 0x61160 +#define DVOC _MMIO(_DVOC) #define DVO_ENABLE (1 << 31) #define DVO_PIPE_B_SELECT (1 << 30) #define DVO_PIPE_STALL_UNUSED (0 << 28) @@ -3274,14 +3456,14 @@ enum skl_disp_power_wells { #define DVO_OUTPUT_CSTATE_PIXELS (1 << 1) /* SDG only */ #define DVO_OUTPUT_SOURCE_SIZE_PIXELS (1 << 0) /* SDG only */ #define DVO_PRESERVE_MASK (0x7<<24) -#define DVOA_SRCDIM 0x61124 -#define DVOB_SRCDIM 0x61144 -#define DVOC_SRCDIM 0x61164 +#define DVOA_SRCDIM _MMIO(0x61124) +#define DVOB_SRCDIM _MMIO(0x61144) +#define DVOC_SRCDIM _MMIO(0x61164) #define DVO_SRCDIM_HORIZONTAL_SHIFT 12 #define DVO_SRCDIM_VERTICAL_SHIFT 0 /* LVDS port control */ -#define LVDS 0x61180 +#define LVDS _MMIO(0x61180) /* * Enables the LVDS port. This bit must be set before DPLLs are enabled, as * the DPLL semantics change when the LVDS is assigned to that pipe. @@ -3331,13 +3513,13 @@ enum skl_disp_power_wells { #define LVDS_B0B3_POWER_UP (3 << 2) /* Video Data Island Packet control */ -#define VIDEO_DIP_DATA 0x61178 +#define VIDEO_DIP_DATA _MMIO(0x61178) /* Read the description of VIDEO_DIP_DATA (before Haswell) or VIDEO_DIP_ECC * (Haswell and newer) to see which VIDEO_DIP_DATA byte corresponds to each byte * of the infoframe structure specified by CEA-861. */ #define VIDEO_DIP_DATA_SIZE 32 #define VIDEO_DIP_VSC_DATA_SIZE 36 -#define VIDEO_DIP_CTL 0x61170 +#define VIDEO_DIP_CTL _MMIO(0x61170) /* Pre HSW: */ #define VIDEO_DIP_ENABLE (1 << 31) #define VIDEO_DIP_PORT(port) ((port) << 29) @@ -3364,7 +3546,7 @@ enum skl_disp_power_wells { #define VIDEO_DIP_ENABLE_SPD_HSW (1 << 0) /* Panel power sequencing */ -#define PP_STATUS 0x61200 +#define PP_STATUS _MMIO(0x61200) #define PP_ON (1 << 31) /* * Indicates that all dependencies of the panel are on: @@ -3390,14 +3572,14 @@ enum skl_disp_power_wells { #define PP_SEQUENCE_STATE_ON_S1_2 (0xa << 0) #define PP_SEQUENCE_STATE_ON_S1_3 (0xb << 0) #define PP_SEQUENCE_STATE_RESET (0xf << 0) -#define PP_CONTROL 0x61204 +#define PP_CONTROL _MMIO(0x61204) #define POWER_TARGET_ON (1 << 0) -#define PP_ON_DELAYS 0x61208 -#define PP_OFF_DELAYS 0x6120c -#define PP_DIVISOR 0x61210 +#define PP_ON_DELAYS _MMIO(0x61208) +#define PP_OFF_DELAYS _MMIO(0x6120c) +#define PP_DIVISOR _MMIO(0x61210) /* Panel fitting */ -#define PFIT_CONTROL (dev_priv->info.display_mmio_offset + 0x61230) +#define PFIT_CONTROL _MMIO(dev_priv->info.display_mmio_offset + 0x61230) #define PFIT_ENABLE (1 << 31) #define PFIT_PIPE_MASK (3 << 29) #define PFIT_PIPE_SHIFT 29 @@ -3415,7 +3597,7 @@ enum skl_disp_power_wells { #define PFIT_SCALING_PROGRAMMED (1 << 26) #define PFIT_SCALING_PILLAR (2 << 26) #define PFIT_SCALING_LETTER (3 << 26) -#define PFIT_PGM_RATIOS (dev_priv->info.display_mmio_offset + 0x61234) +#define PFIT_PGM_RATIOS _MMIO(dev_priv->info.display_mmio_offset + 0x61234) /* Pre-965 */ #define PFIT_VERT_SCALE_SHIFT 20 #define PFIT_VERT_SCALE_MASK 0xfff00000 @@ -3427,25 +3609,25 @@ enum skl_disp_power_wells { #define PFIT_HORIZ_SCALE_SHIFT_965 0 #define PFIT_HORIZ_SCALE_MASK_965 0x00001fff -#define PFIT_AUTO_RATIOS (dev_priv->info.display_mmio_offset + 0x61238) +#define PFIT_AUTO_RATIOS _MMIO(dev_priv->info.display_mmio_offset + 0x61238) #define _VLV_BLC_PWM_CTL2_A (dev_priv->info.display_mmio_offset + 0x61250) #define _VLV_BLC_PWM_CTL2_B (dev_priv->info.display_mmio_offset + 0x61350) -#define VLV_BLC_PWM_CTL2(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ - _VLV_BLC_PWM_CTL2_B) +#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ + _VLV_BLC_PWM_CTL2_B) #define _VLV_BLC_PWM_CTL_A (dev_priv->info.display_mmio_offset + 0x61254) #define _VLV_BLC_PWM_CTL_B (dev_priv->info.display_mmio_offset + 0x61354) -#define VLV_BLC_PWM_CTL(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ - _VLV_BLC_PWM_CTL_B) +#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ + _VLV_BLC_PWM_CTL_B) #define _VLV_BLC_HIST_CTL_A (dev_priv->info.display_mmio_offset + 0x61260) #define _VLV_BLC_HIST_CTL_B (dev_priv->info.display_mmio_offset + 0x61360) -#define VLV_BLC_HIST_CTL(pipe) _PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ - _VLV_BLC_HIST_CTL_B) +#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ + _VLV_BLC_HIST_CTL_B) /* Backlight control */ -#define BLC_PWM_CTL2 (dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */ +#define BLC_PWM_CTL2 _MMIO(dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */ #define BLM_PWM_ENABLE (1 << 31) #define BLM_COMBINATION_MODE (1 << 30) /* gen4 only */ #define BLM_PIPE_SELECT (1 << 29) @@ -3468,7 +3650,7 @@ enum skl_disp_power_wells { #define BLM_PHASE_IN_COUNT_MASK (0xff << 8) #define BLM_PHASE_IN_INCR_SHIFT (0) #define BLM_PHASE_IN_INCR_MASK (0xff << 0) -#define BLC_PWM_CTL (dev_priv->info.display_mmio_offset + 0x61254) +#define BLC_PWM_CTL _MMIO(dev_priv->info.display_mmio_offset + 0x61254) /* * This is the most significant 15 bits of the number of backlight cycles in a * complete cycle of the modulated backlight control. @@ -3490,44 +3672,56 @@ enum skl_disp_power_wells { #define BACKLIGHT_DUTY_CYCLE_MASK_PNV (0xfffe) #define BLM_POLARITY_PNV (1 << 0) /* pnv only */ -#define BLC_HIST_CTL (dev_priv->info.display_mmio_offset + 0x61260) +#define BLC_HIST_CTL _MMIO(dev_priv->info.display_mmio_offset + 0x61260) #define BLM_HISTOGRAM_ENABLE (1 << 31) /* New registers for PCH-split platforms. Safe where new bits show up, the * register layout machtes with gen4 BLC_PWM_CTL[12]. */ -#define BLC_PWM_CPU_CTL2 0x48250 -#define BLC_PWM_CPU_CTL 0x48254 +#define BLC_PWM_CPU_CTL2 _MMIO(0x48250) +#define BLC_PWM_CPU_CTL _MMIO(0x48254) -#define HSW_BLC_PWM2_CTL 0x48350 +#define HSW_BLC_PWM2_CTL _MMIO(0x48350) /* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */ -#define BLC_PWM_PCH_CTL1 0xc8250 +#define BLC_PWM_PCH_CTL1 _MMIO(0xc8250) #define BLM_PCH_PWM_ENABLE (1 << 31) #define BLM_PCH_OVERRIDE_ENABLE (1 << 30) #define BLM_PCH_POLARITY (1 << 29) -#define BLC_PWM_PCH_CTL2 0xc8254 +#define BLC_PWM_PCH_CTL2 _MMIO(0xc8254) -#define UTIL_PIN_CTL 0x48400 +#define UTIL_PIN_CTL _MMIO(0x48400) #define UTIL_PIN_ENABLE (1 << 31) +#define UTIL_PIN_PIPE(x) ((x) << 29) +#define UTIL_PIN_PIPE_MASK (3 << 29) +#define UTIL_PIN_MODE_PWM (1 << 24) +#define UTIL_PIN_MODE_MASK (0xf << 24) +#define UTIL_PIN_POLARITY (1 << 22) + /* BXT backlight register definition. */ -#define BXT_BLC_PWM_CTL1 0xC8250 +#define _BXT_BLC_PWM_CTL1 0xC8250 #define BXT_BLC_PWM_ENABLE (1 << 31) #define BXT_BLC_PWM_POLARITY (1 << 29) -#define BXT_BLC_PWM_FREQ1 0xC8254 -#define BXT_BLC_PWM_DUTY1 0xC8258 +#define _BXT_BLC_PWM_FREQ1 0xC8254 +#define _BXT_BLC_PWM_DUTY1 0xC8258 -#define BXT_BLC_PWM_CTL2 0xC8350 -#define BXT_BLC_PWM_FREQ2 0xC8354 -#define BXT_BLC_PWM_DUTY2 0xC8358 +#define _BXT_BLC_PWM_CTL2 0xC8350 +#define _BXT_BLC_PWM_FREQ2 0xC8354 +#define _BXT_BLC_PWM_DUTY2 0xC8358 +#define BXT_BLC_PWM_CTL(controller) _MMIO_PIPE(controller, \ + _BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2) +#define BXT_BLC_PWM_FREQ(controller) _MMIO_PIPE(controller, \ + _BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2) +#define BXT_BLC_PWM_DUTY(controller) _MMIO_PIPE(controller, \ + _BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2) -#define PCH_GTC_CTL 0xe7000 +#define PCH_GTC_CTL _MMIO(0xe7000) #define PCH_GTC_ENABLE (1 << 31) /* TV port control */ -#define TV_CTL 0x68000 +#define TV_CTL _MMIO(0x68000) /* Enables the TV encoder */ # define TV_ENC_ENABLE (1 << 31) /* Sources the TV encoder input from pipe B instead of A. */ @@ -3594,7 +3788,7 @@ enum skl_disp_power_wells { # define TV_TEST_MODE_MONITOR_DETECT (7 << 0) # define TV_TEST_MODE_MASK (7 << 0) -#define TV_DAC 0x68004 +#define TV_DAC _MMIO(0x68004) # define TV_DAC_SAVE 0x00ffff00 /* * Reports that DAC state change logic has reported change (RO). @@ -3645,13 +3839,13 @@ enum skl_disp_power_wells { * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with * -1 (0x3) being the only legal negative value. */ -#define TV_CSC_Y 0x68010 +#define TV_CSC_Y _MMIO(0x68010) # define TV_RY_MASK 0x07ff0000 # define TV_RY_SHIFT 16 # define TV_GY_MASK 0x00000fff # define TV_GY_SHIFT 0 -#define TV_CSC_Y2 0x68014 +#define TV_CSC_Y2 _MMIO(0x68014) # define TV_BY_MASK 0x07ff0000 # define TV_BY_SHIFT 16 /* @@ -3662,13 +3856,13 @@ enum skl_disp_power_wells { # define TV_AY_MASK 0x000003ff # define TV_AY_SHIFT 0 -#define TV_CSC_U 0x68018 +#define TV_CSC_U _MMIO(0x68018) # define TV_RU_MASK 0x07ff0000 # define TV_RU_SHIFT 16 # define TV_GU_MASK 0x000007ff # define TV_GU_SHIFT 0 -#define TV_CSC_U2 0x6801c +#define TV_CSC_U2 _MMIO(0x6801c) # define TV_BU_MASK 0x07ff0000 # define TV_BU_SHIFT 16 /* @@ -3679,13 +3873,13 @@ enum skl_disp_power_wells { # define TV_AU_MASK 0x000003ff # define TV_AU_SHIFT 0 -#define TV_CSC_V 0x68020 +#define TV_CSC_V _MMIO(0x68020) # define TV_RV_MASK 0x0fff0000 # define TV_RV_SHIFT 16 # define TV_GV_MASK 0x000007ff # define TV_GV_SHIFT 0 -#define TV_CSC_V2 0x68024 +#define TV_CSC_V2 _MMIO(0x68024) # define TV_BV_MASK 0x07ff0000 # define TV_BV_SHIFT 16 /* @@ -3696,7 +3890,7 @@ enum skl_disp_power_wells { # define TV_AV_MASK 0x000007ff # define TV_AV_SHIFT 0 -#define TV_CLR_KNOBS 0x68028 +#define TV_CLR_KNOBS _MMIO(0x68028) /* 2s-complement brightness adjustment */ # define TV_BRIGHTNESS_MASK 0xff000000 # define TV_BRIGHTNESS_SHIFT 24 @@ -3710,7 +3904,7 @@ enum skl_disp_power_wells { # define TV_HUE_MASK 0x000000ff # define TV_HUE_SHIFT 0 -#define TV_CLR_LEVEL 0x6802c +#define TV_CLR_LEVEL _MMIO(0x6802c) /* Controls the DAC level for black */ # define TV_BLACK_LEVEL_MASK 0x01ff0000 # define TV_BLACK_LEVEL_SHIFT 16 @@ -3718,7 +3912,7 @@ enum skl_disp_power_wells { # define TV_BLANK_LEVEL_MASK 0x000001ff # define TV_BLANK_LEVEL_SHIFT 0 -#define TV_H_CTL_1 0x68030 +#define TV_H_CTL_1 _MMIO(0x68030) /* Number of pixels in the hsync. */ # define TV_HSYNC_END_MASK 0x1fff0000 # define TV_HSYNC_END_SHIFT 16 @@ -3726,7 +3920,7 @@ enum skl_disp_power_wells { # define TV_HTOTAL_MASK 0x00001fff # define TV_HTOTAL_SHIFT 0 -#define TV_H_CTL_2 0x68034 +#define TV_H_CTL_2 _MMIO(0x68034) /* Enables the colorburst (needed for non-component color) */ # define TV_BURST_ENA (1 << 31) /* Offset of the colorburst from the start of hsync, in pixels minus one. */ @@ -3736,7 +3930,7 @@ enum skl_disp_power_wells { # define TV_HBURST_LEN_SHIFT 0 # define TV_HBURST_LEN_MASK 0x0001fff -#define TV_H_CTL_3 0x68038 +#define TV_H_CTL_3 _MMIO(0x68038) /* End of hblank, measured in pixels minus one from start of hsync */ # define TV_HBLANK_END_SHIFT 16 # define TV_HBLANK_END_MASK 0x1fff0000 @@ -3744,7 +3938,7 @@ enum skl_disp_power_wells { # define TV_HBLANK_START_SHIFT 0 # define TV_HBLANK_START_MASK 0x0001fff -#define TV_V_CTL_1 0x6803c +#define TV_V_CTL_1 _MMIO(0x6803c) /* XXX */ # define TV_NBR_END_SHIFT 16 # define TV_NBR_END_MASK 0x07ff0000 @@ -3755,7 +3949,7 @@ enum skl_disp_power_wells { # define TV_VI_END_F2_SHIFT 0 # define TV_VI_END_F2_MASK 0x0000003f -#define TV_V_CTL_2 0x68040 +#define TV_V_CTL_2 _MMIO(0x68040) /* Length of vsync, in half lines */ # define TV_VSYNC_LEN_MASK 0x07ff0000 # define TV_VSYNC_LEN_SHIFT 16 @@ -3771,7 +3965,7 @@ enum skl_disp_power_wells { # define TV_VSYNC_START_F2_MASK 0x0000007f # define TV_VSYNC_START_F2_SHIFT 0 -#define TV_V_CTL_3 0x68044 +#define TV_V_CTL_3 _MMIO(0x68044) /* Enables generation of the equalization signal */ # define TV_EQUAL_ENA (1 << 31) /* Length of vsync, in half lines */ @@ -3789,7 +3983,7 @@ enum skl_disp_power_wells { # define TV_VEQ_START_F2_MASK 0x000007f # define TV_VEQ_START_F2_SHIFT 0 -#define TV_V_CTL_4 0x68048 +#define TV_V_CTL_4 _MMIO(0x68048) /* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. @@ -3803,7 +3997,7 @@ enum skl_disp_power_wells { # define TV_VBURST_END_F1_MASK 0x000000ff # define TV_VBURST_END_F1_SHIFT 0 -#define TV_V_CTL_5 0x6804c +#define TV_V_CTL_5 _MMIO(0x6804c) /* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. @@ -3817,7 +4011,7 @@ enum skl_disp_power_wells { # define TV_VBURST_END_F2_MASK 0x000000ff # define TV_VBURST_END_F2_SHIFT 0 -#define TV_V_CTL_6 0x68050 +#define TV_V_CTL_6 _MMIO(0x68050) /* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. @@ -3831,7 +4025,7 @@ enum skl_disp_power_wells { # define TV_VBURST_END_F3_MASK 0x000000ff # define TV_VBURST_END_F3_SHIFT 0 -#define TV_V_CTL_7 0x68054 +#define TV_V_CTL_7 _MMIO(0x68054) /* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. @@ -3845,7 +4039,7 @@ enum skl_disp_power_wells { # define TV_VBURST_END_F4_MASK 0x000000ff # define TV_VBURST_END_F4_SHIFT 0 -#define TV_SC_CTL_1 0x68060 +#define TV_SC_CTL_1 _MMIO(0x68060) /* Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA1_EN (1 << 31) /* Turns on the first subcarrier phase generation DDA */ @@ -3867,7 +4061,7 @@ enum skl_disp_power_wells { # define TV_SCDDA1_INC_MASK 0x00000fff # define TV_SCDDA1_INC_SHIFT 0 -#define TV_SC_CTL_2 0x68064 +#define TV_SC_CTL_2 _MMIO(0x68064) /* Sets the rollover for the second subcarrier phase generation DDA */ # define TV_SCDDA2_SIZE_MASK 0x7fff0000 # define TV_SCDDA2_SIZE_SHIFT 16 @@ -3875,7 +4069,7 @@ enum skl_disp_power_wells { # define TV_SCDDA2_INC_MASK 0x00007fff # define TV_SCDDA2_INC_SHIFT 0 -#define TV_SC_CTL_3 0x68068 +#define TV_SC_CTL_3 _MMIO(0x68068) /* Sets the rollover for the third subcarrier phase generation DDA */ # define TV_SCDDA3_SIZE_MASK 0x7fff0000 # define TV_SCDDA3_SIZE_SHIFT 16 @@ -3883,7 +4077,7 @@ enum skl_disp_power_wells { # define TV_SCDDA3_INC_MASK 0x00007fff # define TV_SCDDA3_INC_SHIFT 0 -#define TV_WIN_POS 0x68070 +#define TV_WIN_POS _MMIO(0x68070) /* X coordinate of the display from the start of horizontal active */ # define TV_XPOS_MASK 0x1fff0000 # define TV_XPOS_SHIFT 16 @@ -3891,7 +4085,7 @@ enum skl_disp_power_wells { # define TV_YPOS_MASK 0x00000fff # define TV_YPOS_SHIFT 0 -#define TV_WIN_SIZE 0x68074 +#define TV_WIN_SIZE _MMIO(0x68074) /* Horizontal size of the display window, measured in pixels*/ # define TV_XSIZE_MASK 0x1fff0000 # define TV_XSIZE_SHIFT 16 @@ -3903,7 +4097,7 @@ enum skl_disp_power_wells { # define TV_YSIZE_MASK 0x00000fff # define TV_YSIZE_SHIFT 0 -#define TV_FILTER_CTL_1 0x68080 +#define TV_FILTER_CTL_1 _MMIO(0x68080) /* * Enables automatic scaling calculation. * @@ -3936,7 +4130,7 @@ enum skl_disp_power_wells { # define TV_HSCALE_FRAC_MASK 0x00003fff # define TV_HSCALE_FRAC_SHIFT 0 -#define TV_FILTER_CTL_2 0x68084 +#define TV_FILTER_CTL_2 _MMIO(0x68084) /* * Sets the integer part of the 3.15 fixed-point vertical scaling factor. * @@ -3952,7 +4146,7 @@ enum skl_disp_power_wells { # define TV_VSCALE_FRAC_MASK 0x00007fff # define TV_VSCALE_FRAC_SHIFT 0 -#define TV_FILTER_CTL_3 0x68088 +#define TV_FILTER_CTL_3 _MMIO(0x68088) /* * Sets the integer part of the 3.15 fixed-point vertical scaling factor. * @@ -3972,7 +4166,7 @@ enum skl_disp_power_wells { # define TV_VSCALE_IP_FRAC_MASK 0x00007fff # define TV_VSCALE_IP_FRAC_SHIFT 0 -#define TV_CC_CONTROL 0x68090 +#define TV_CC_CONTROL _MMIO(0x68090) # define TV_CC_ENABLE (1 << 31) /* * Specifies which field to send the CC data in. @@ -3988,7 +4182,7 @@ enum skl_disp_power_wells { # define TV_CC_LINE_MASK 0x0000003f # define TV_CC_LINE_SHIFT 0 -#define TV_CC_DATA 0x68094 +#define TV_CC_DATA _MMIO(0x68094) # define TV_CC_RDY (1 << 31) /* Second word of CC data to be transmitted. */ # define TV_CC_DATA_2_MASK 0x007f0000 @@ -3997,20 +4191,20 @@ enum skl_disp_power_wells { # define TV_CC_DATA_1_MASK 0x0000007f # define TV_CC_DATA_1_SHIFT 0 -#define TV_H_LUMA_0 0x68100 -#define TV_H_LUMA_59 0x681ec -#define TV_H_CHROMA_0 0x68200 -#define TV_H_CHROMA_59 0x682ec -#define TV_V_LUMA_0 0x68300 -#define TV_V_LUMA_42 0x683a8 -#define TV_V_CHROMA_0 0x68400 -#define TV_V_CHROMA_42 0x684a8 +#define TV_H_LUMA(i) _MMIO(0x68100 + (i) * 4) /* 60 registers */ +#define TV_H_CHROMA(i) _MMIO(0x68200 + (i) * 4) /* 60 registers */ +#define TV_V_LUMA(i) _MMIO(0x68300 + (i) * 4) /* 43 registers */ +#define TV_V_CHROMA(i) _MMIO(0x68400 + (i) * 4) /* 43 registers */ /* Display Port */ -#define DP_A 0x64000 /* eDP */ -#define DP_B 0x64100 -#define DP_C 0x64200 -#define DP_D 0x64300 +#define DP_A _MMIO(0x64000) /* eDP */ +#define DP_B _MMIO(0x64100) +#define DP_C _MMIO(0x64200) +#define DP_D _MMIO(0x64300) + +#define VLV_DP_B _MMIO(VLV_DISPLAY_BASE + 0x64100) +#define VLV_DP_C _MMIO(VLV_DISPLAY_BASE + 0x64200) +#define CHV_DP_D _MMIO(VLV_DISPLAY_BASE + 0x64300) #define DP_PORT_EN (1 << 31) #define DP_PIPEB_SELECT (1 << 30) @@ -4057,13 +4251,14 @@ enum skl_disp_power_wells { /* How many wires to use. I guess 3 was too hard */ #define DP_PORT_WIDTH(width) (((width) - 1) << 19) #define DP_PORT_WIDTH_MASK (7 << 19) +#define DP_PORT_WIDTH_SHIFT 19 /* Mystic DPCD version 1.1 special mode */ #define DP_ENHANCED_FRAMING (1 << 18) /* eDP */ #define DP_PLL_FREQ_270MHZ (0 << 16) -#define DP_PLL_FREQ_160MHZ (1 << 16) +#define DP_PLL_FREQ_162MHZ (1 << 16) #define DP_PLL_FREQ_MASK (3 << 16) /* locked once port is enabled */ @@ -4096,33 +4291,36 @@ enum skl_disp_power_wells { * is 20 bytes in each direction, hence the 5 fixed * data registers */ -#define DPA_AUX_CH_CTL 0x64010 -#define DPA_AUX_CH_DATA1 0x64014 -#define DPA_AUX_CH_DATA2 0x64018 -#define DPA_AUX_CH_DATA3 0x6401c -#define DPA_AUX_CH_DATA4 0x64020 -#define DPA_AUX_CH_DATA5 0x64024 - -#define DPB_AUX_CH_CTL 0x64110 -#define DPB_AUX_CH_DATA1 0x64114 -#define DPB_AUX_CH_DATA2 0x64118 -#define DPB_AUX_CH_DATA3 0x6411c -#define DPB_AUX_CH_DATA4 0x64120 -#define DPB_AUX_CH_DATA5 0x64124 - -#define DPC_AUX_CH_CTL 0x64210 -#define DPC_AUX_CH_DATA1 0x64214 -#define DPC_AUX_CH_DATA2 0x64218 -#define DPC_AUX_CH_DATA3 0x6421c -#define DPC_AUX_CH_DATA4 0x64220 -#define DPC_AUX_CH_DATA5 0x64224 - -#define DPD_AUX_CH_CTL 0x64310 -#define DPD_AUX_CH_DATA1 0x64314 -#define DPD_AUX_CH_DATA2 0x64318 -#define DPD_AUX_CH_DATA3 0x6431c -#define DPD_AUX_CH_DATA4 0x64320 -#define DPD_AUX_CH_DATA5 0x64324 +#define _DPA_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64010) +#define _DPA_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64014) +#define _DPA_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64018) +#define _DPA_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6401c) +#define _DPA_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64020) +#define _DPA_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64024) + +#define _DPB_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64110) +#define _DPB_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64114) +#define _DPB_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64118) +#define _DPB_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6411c) +#define _DPB_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64120) +#define _DPB_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64124) + +#define _DPC_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64210) +#define _DPC_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64214) +#define _DPC_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64218) +#define _DPC_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6421c) +#define _DPC_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64220) +#define _DPC_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64224) + +#define _DPD_AUX_CH_CTL (dev_priv->info.display_mmio_offset + 0x64310) +#define _DPD_AUX_CH_DATA1 (dev_priv->info.display_mmio_offset + 0x64314) +#define _DPD_AUX_CH_DATA2 (dev_priv->info.display_mmio_offset + 0x64318) +#define _DPD_AUX_CH_DATA3 (dev_priv->info.display_mmio_offset + 0x6431c) +#define _DPD_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64320) +#define _DPD_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64324) + +#define DP_AUX_CH_CTL(port) _MMIO_PORT(port, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL) +#define DP_AUX_CH_DATA(port, i) _MMIO(_PORT(port, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ #define DP_AUX_CH_CTL_SEND_BUSY (1 << 31) #define DP_AUX_CH_CTL_DONE (1 << 30) @@ -4148,7 +4346,7 @@ enum skl_disp_power_wells { #define DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL (1 << 14) #define DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL (1 << 13) #define DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL (1 << 12) -#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (1f << 5) +#define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL_MASK (0x1f << 5) #define DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(c) (((c) - 1) << 5) #define DP_AUX_CH_CTL_SYNC_PULSE_SKL(c) ((c) - 1) @@ -4199,10 +4397,10 @@ enum skl_disp_power_wells { #define _PIPEB_LINK_N_G4X 0x71064 #define PIPEA_DP_LINK_N_MASK (0xffffff) -#define PIPE_DATA_M_G4X(pipe) _PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X) -#define PIPE_DATA_N_G4X(pipe) _PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X) -#define PIPE_LINK_M_G4X(pipe) _PIPE(pipe, _PIPEA_LINK_M_G4X, _PIPEB_LINK_M_G4X) -#define PIPE_LINK_N_G4X(pipe) _PIPE(pipe, _PIPEA_LINK_N_G4X, _PIPEB_LINK_N_G4X) +#define PIPE_DATA_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X) +#define PIPE_DATA_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X) +#define PIPE_LINK_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_LINK_M_G4X, _PIPEB_LINK_M_G4X) +#define PIPE_LINK_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_LINK_N_G4X, _PIPEB_LINK_N_G4X) /* Display & cursor control */ @@ -4318,15 +4516,15 @@ enum skl_disp_power_wells { */ #define PIPE_EDP_OFFSET 0x7f000 -#define _PIPE2(pipe, reg) (dev_priv->info.pipe_offsets[pipe] - \ +#define _MMIO_PIPE2(pipe, reg) _MMIO(dev_priv->info.pipe_offsets[pipe] - \ dev_priv->info.pipe_offsets[PIPE_A] + (reg) + \ dev_priv->info.display_mmio_offset) -#define PIPECONF(pipe) _PIPE2(pipe, _PIPEACONF) -#define PIPEDSL(pipe) _PIPE2(pipe, _PIPEADSL) -#define PIPEFRAME(pipe) _PIPE2(pipe, _PIPEAFRAMEHIGH) -#define PIPEFRAMEPIXEL(pipe) _PIPE2(pipe, _PIPEAFRAMEPIXEL) -#define PIPESTAT(pipe) _PIPE2(pipe, _PIPEASTAT) +#define PIPECONF(pipe) _MMIO_PIPE2(pipe, _PIPEACONF) +#define PIPEDSL(pipe) _MMIO_PIPE2(pipe, _PIPEADSL) +#define PIPEFRAME(pipe) _MMIO_PIPE2(pipe, _PIPEAFRAMEHIGH) +#define PIPEFRAMEPIXEL(pipe) _MMIO_PIPE2(pipe, _PIPEAFRAMEPIXEL) +#define PIPESTAT(pipe) _MMIO_PIPE2(pipe, _PIPEASTAT) #define _PIPE_MISC_A 0x70030 #define _PIPE_MISC_B 0x71030 @@ -4338,9 +4536,9 @@ enum skl_disp_power_wells { #define PIPEMISC_DITHER_ENABLE (1<<4) #define PIPEMISC_DITHER_TYPE_MASK (3<<2) #define PIPEMISC_DITHER_TYPE_SP (0<<2) -#define PIPEMISC(pipe) _PIPE2(pipe, _PIPE_MISC_A) +#define PIPEMISC(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC_A) -#define VLV_DPFLIPSTAT (VLV_DISPLAY_BASE + 0x70028) +#define VLV_DPFLIPSTAT _MMIO(VLV_DISPLAY_BASE + 0x70028) #define PIPEB_LINE_COMPARE_INT_EN (1<<29) #define PIPEB_HLINE_INT_EN (1<<28) #define PIPEB_VBLANK_INT_EN (1<<27) @@ -4361,7 +4559,7 @@ enum skl_disp_power_wells { #define SPRITEE_FLIPDONE_INT_EN (1<<9) #define PLANEC_FLIPDONE_INT_EN (1<<8) -#define DPINVGTT (VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */ +#define DPINVGTT _MMIO(VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */ #define SPRITEF_INVALID_GTT_INT_EN (1<<27) #define SPRITEE_INVALID_GTT_INT_EN (1<<26) #define PLANEC_INVALID_GTT_INT_EN (1<<25) @@ -4391,19 +4589,42 @@ enum skl_disp_power_wells { #define DPINVGTT_STATUS_MASK 0xff #define DPINVGTT_STATUS_MASK_CHV 0xfff -#define DSPARB (dev_priv->info.display_mmio_offset + 0x70030) +#define DSPARB _MMIO(dev_priv->info.display_mmio_offset + 0x70030) #define DSPARB_CSTART_MASK (0x7f << 7) #define DSPARB_CSTART_SHIFT 7 #define DSPARB_BSTART_MASK (0x7f) #define DSPARB_BSTART_SHIFT 0 #define DSPARB_BEND_SHIFT 9 /* on 855 */ #define DSPARB_AEND_SHIFT 0 - -#define DSPARB2 (VLV_DISPLAY_BASE + 0x70060) /* vlv/chv */ -#define DSPARB3 (VLV_DISPLAY_BASE + 0x7006c) /* chv */ +#define DSPARB_SPRITEA_SHIFT_VLV 0 +#define DSPARB_SPRITEA_MASK_VLV (0xff << 0) +#define DSPARB_SPRITEB_SHIFT_VLV 8 +#define DSPARB_SPRITEB_MASK_VLV (0xff << 8) +#define DSPARB_SPRITEC_SHIFT_VLV 16 +#define DSPARB_SPRITEC_MASK_VLV (0xff << 16) +#define DSPARB_SPRITED_SHIFT_VLV 24 +#define DSPARB_SPRITED_MASK_VLV (0xff << 24) +#define DSPARB2 _MMIO(VLV_DISPLAY_BASE + 0x70060) /* vlv/chv */ +#define DSPARB_SPRITEA_HI_SHIFT_VLV 0 +#define DSPARB_SPRITEA_HI_MASK_VLV (0x1 << 0) +#define DSPARB_SPRITEB_HI_SHIFT_VLV 4 +#define DSPARB_SPRITEB_HI_MASK_VLV (0x1 << 4) +#define DSPARB_SPRITEC_HI_SHIFT_VLV 8 +#define DSPARB_SPRITEC_HI_MASK_VLV (0x1 << 8) +#define DSPARB_SPRITED_HI_SHIFT_VLV 12 +#define DSPARB_SPRITED_HI_MASK_VLV (0x1 << 12) +#define DSPARB_SPRITEE_HI_SHIFT_VLV 16 +#define DSPARB_SPRITEE_HI_MASK_VLV (0x1 << 16) +#define DSPARB_SPRITEF_HI_SHIFT_VLV 20 +#define DSPARB_SPRITEF_HI_MASK_VLV (0x1 << 20) +#define DSPARB3 _MMIO(VLV_DISPLAY_BASE + 0x7006c) /* chv */ +#define DSPARB_SPRITEE_SHIFT_VLV 0 +#define DSPARB_SPRITEE_MASK_VLV (0xff << 0) +#define DSPARB_SPRITEF_SHIFT_VLV 8 +#define DSPARB_SPRITEF_MASK_VLV (0xff << 8) /* pnv/gen4/g4x/vlv/chv */ -#define DSPFW1 (dev_priv->info.display_mmio_offset + 0x70034) +#define DSPFW1 _MMIO(dev_priv->info.display_mmio_offset + 0x70034) #define DSPFW_SR_SHIFT 23 #define DSPFW_SR_MASK (0x1ff<<23) #define DSPFW_CURSORB_SHIFT 16 @@ -4414,7 +4635,7 @@ enum skl_disp_power_wells { #define DSPFW_PLANEA_SHIFT 0 #define DSPFW_PLANEA_MASK (0x7f<<0) #define DSPFW_PLANEA_MASK_VLV (0xff<<0) /* vlv/chv */ -#define DSPFW2 (dev_priv->info.display_mmio_offset + 0x70038) +#define DSPFW2 _MMIO(dev_priv->info.display_mmio_offset + 0x70038) #define DSPFW_FBC_SR_EN (1<<31) /* g4x */ #define DSPFW_FBC_SR_SHIFT 28 #define DSPFW_FBC_SR_MASK (0x7<<28) /* g4x */ @@ -4430,7 +4651,7 @@ enum skl_disp_power_wells { #define DSPFW_SPRITEA_SHIFT 0 #define DSPFW_SPRITEA_MASK (0x7f<<0) /* g4x */ #define DSPFW_SPRITEA_MASK_VLV (0xff<<0) /* vlv/chv */ -#define DSPFW3 (dev_priv->info.display_mmio_offset + 0x7003c) +#define DSPFW3 _MMIO(dev_priv->info.display_mmio_offset + 0x7003c) #define DSPFW_HPLL_SR_EN (1<<31) #define PINEVIEW_SELF_REFRESH_EN (1<<30) #define DSPFW_CURSOR_SR_SHIFT 24 @@ -4441,14 +4662,14 @@ enum skl_disp_power_wells { #define DSPFW_HPLL_SR_MASK (0x1ff<<0) /* vlv/chv */ -#define DSPFW4 (VLV_DISPLAY_BASE + 0x70070) +#define DSPFW4 _MMIO(VLV_DISPLAY_BASE + 0x70070) #define DSPFW_SPRITEB_WM1_SHIFT 16 #define DSPFW_SPRITEB_WM1_MASK (0xff<<16) #define DSPFW_CURSORA_WM1_SHIFT 8 #define DSPFW_CURSORA_WM1_MASK (0x3f<<8) #define DSPFW_SPRITEA_WM1_SHIFT 0 #define DSPFW_SPRITEA_WM1_MASK (0xff<<0) -#define DSPFW5 (VLV_DISPLAY_BASE + 0x70074) +#define DSPFW5 _MMIO(VLV_DISPLAY_BASE + 0x70074) #define DSPFW_PLANEB_WM1_SHIFT 24 #define DSPFW_PLANEB_WM1_MASK (0xff<<24) #define DSPFW_PLANEA_WM1_SHIFT 16 @@ -4457,11 +4678,11 @@ enum skl_disp_power_wells { #define DSPFW_CURSORB_WM1_MASK (0x3f<<8) #define DSPFW_CURSOR_SR_WM1_SHIFT 0 #define DSPFW_CURSOR_SR_WM1_MASK (0x3f<<0) -#define DSPFW6 (VLV_DISPLAY_BASE + 0x70078) +#define DSPFW6 _MMIO(VLV_DISPLAY_BASE + 0x70078) #define DSPFW_SR_WM1_SHIFT 0 #define DSPFW_SR_WM1_MASK (0x1ff<<0) -#define DSPFW7 (VLV_DISPLAY_BASE + 0x7007c) -#define DSPFW7_CHV (VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */ +#define DSPFW7 _MMIO(VLV_DISPLAY_BASE + 0x7007c) +#define DSPFW7_CHV _MMIO(VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */ #define DSPFW_SPRITED_WM1_SHIFT 24 #define DSPFW_SPRITED_WM1_MASK (0xff<<24) #define DSPFW_SPRITED_SHIFT 16 @@ -4470,7 +4691,7 @@ enum skl_disp_power_wells { #define DSPFW_SPRITEC_WM1_MASK (0xff<<8) #define DSPFW_SPRITEC_SHIFT 0 #define DSPFW_SPRITEC_MASK_VLV (0xff<<0) -#define DSPFW8_CHV (VLV_DISPLAY_BASE + 0x700b8) +#define DSPFW8_CHV _MMIO(VLV_DISPLAY_BASE + 0x700b8) #define DSPFW_SPRITEF_WM1_SHIFT 24 #define DSPFW_SPRITEF_WM1_MASK (0xff<<24) #define DSPFW_SPRITEF_SHIFT 16 @@ -4479,7 +4700,7 @@ enum skl_disp_power_wells { #define DSPFW_SPRITEE_WM1_MASK (0xff<<8) #define DSPFW_SPRITEE_SHIFT 0 #define DSPFW_SPRITEE_MASK_VLV (0xff<<0) -#define DSPFW9_CHV (VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */ +#define DSPFW9_CHV _MMIO(VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */ #define DSPFW_PLANEC_WM1_SHIFT 24 #define DSPFW_PLANEC_WM1_MASK (0xff<<24) #define DSPFW_PLANEC_SHIFT 16 @@ -4490,7 +4711,7 @@ enum skl_disp_power_wells { #define DSPFW_CURSORC_MASK (0x3f<<0) /* vlv/chv high order bits */ -#define DSPHOWM (VLV_DISPLAY_BASE + 0x70064) +#define DSPHOWM _MMIO(VLV_DISPLAY_BASE + 0x70064) #define DSPFW_SR_HI_SHIFT 24 #define DSPFW_SR_HI_MASK (3<<24) /* 2 bits for chv, 1 for vlv */ #define DSPFW_SPRITEF_HI_SHIFT 23 @@ -4511,7 +4732,7 @@ enum skl_disp_power_wells { #define DSPFW_SPRITEA_HI_MASK (1<<4) #define DSPFW_PLANEA_HI_SHIFT 0 #define DSPFW_PLANEA_HI_MASK (1<<0) -#define DSPHOWM1 (VLV_DISPLAY_BASE + 0x70068) +#define DSPHOWM1 _MMIO(VLV_DISPLAY_BASE + 0x70068) #define DSPFW_SR_WM1_HI_SHIFT 24 #define DSPFW_SR_WM1_HI_MASK (3<<24) /* 2 bits for chv, 1 for vlv */ #define DSPFW_SPRITEF_WM1_HI_SHIFT 23 @@ -4534,7 +4755,7 @@ enum skl_disp_power_wells { #define DSPFW_PLANEA_WM1_HI_MASK (1<<0) /* drain latency register values*/ -#define VLV_DDL(pipe) (VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe)) +#define VLV_DDL(pipe) _MMIO(VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe)) #define DDL_CURSOR_SHIFT 24 #define DDL_SPRITE_SHIFT(sprite) (8+8*(sprite)) #define DDL_PLANE_SHIFT 0 @@ -4542,8 +4763,9 @@ enum skl_disp_power_wells { #define DDL_PRECISION_LOW (0<<7) #define DRAIN_LATENCY_MASK 0x7f -#define CBR1_VLV (VLV_DISPLAY_BASE + 0x70400) +#define CBR1_VLV _MMIO(VLV_DISPLAY_BASE + 0x70400) #define CBR_PND_DEADLINE_DISABLE (1<<31) +#define CBR_PWM_CLOCK_MUX_SELECT (1<<30) /* FIFO watermark sizes etc */ #define G4X_FIFO_LINE_SIZE 64 @@ -4579,51 +4801,51 @@ enum skl_disp_power_wells { #define I965_CURSOR_DFT_WM 8 /* Watermark register definitions for SKL */ -#define CUR_WM_A_0 0x70140 -#define CUR_WM_B_0 0x71140 -#define PLANE_WM_1_A_0 0x70240 -#define PLANE_WM_1_B_0 0x71240 -#define PLANE_WM_2_A_0 0x70340 -#define PLANE_WM_2_B_0 0x71340 -#define PLANE_WM_TRANS_1_A_0 0x70268 -#define PLANE_WM_TRANS_1_B_0 0x71268 -#define PLANE_WM_TRANS_2_A_0 0x70368 -#define PLANE_WM_TRANS_2_B_0 0x71368 -#define CUR_WM_TRANS_A_0 0x70168 -#define CUR_WM_TRANS_B_0 0x71168 +#define _CUR_WM_A_0 0x70140 +#define _CUR_WM_B_0 0x71140 +#define _PLANE_WM_1_A_0 0x70240 +#define _PLANE_WM_1_B_0 0x71240 +#define _PLANE_WM_2_A_0 0x70340 +#define _PLANE_WM_2_B_0 0x71340 +#define _PLANE_WM_TRANS_1_A_0 0x70268 +#define _PLANE_WM_TRANS_1_B_0 0x71268 +#define _PLANE_WM_TRANS_2_A_0 0x70368 +#define _PLANE_WM_TRANS_2_B_0 0x71368 +#define _CUR_WM_TRANS_A_0 0x70168 +#define _CUR_WM_TRANS_B_0 0x71168 #define PLANE_WM_EN (1 << 31) #define PLANE_WM_LINES_SHIFT 14 #define PLANE_WM_LINES_MASK 0x1f #define PLANE_WM_BLOCKS_MASK 0x3ff -#define CUR_WM_0(pipe) _PIPE(pipe, CUR_WM_A_0, CUR_WM_B_0) -#define CUR_WM(pipe, level) (CUR_WM_0(pipe) + ((4) * (level))) -#define CUR_WM_TRANS(pipe) _PIPE(pipe, CUR_WM_TRANS_A_0, CUR_WM_TRANS_B_0) +#define _CUR_WM_0(pipe) _PIPE(pipe, _CUR_WM_A_0, _CUR_WM_B_0) +#define CUR_WM(pipe, level) _MMIO(_CUR_WM_0(pipe) + ((4) * (level))) +#define CUR_WM_TRANS(pipe) _MMIO_PIPE(pipe, _CUR_WM_TRANS_A_0, _CUR_WM_TRANS_B_0) -#define _PLANE_WM_1(pipe) _PIPE(pipe, PLANE_WM_1_A_0, PLANE_WM_1_B_0) -#define _PLANE_WM_2(pipe) _PIPE(pipe, PLANE_WM_2_A_0, PLANE_WM_2_B_0) +#define _PLANE_WM_1(pipe) _PIPE(pipe, _PLANE_WM_1_A_0, _PLANE_WM_1_B_0) +#define _PLANE_WM_2(pipe) _PIPE(pipe, _PLANE_WM_2_A_0, _PLANE_WM_2_B_0) #define _PLANE_WM_BASE(pipe, plane) \ _PLANE(plane, _PLANE_WM_1(pipe), _PLANE_WM_2(pipe)) #define PLANE_WM(pipe, plane, level) \ - (_PLANE_WM_BASE(pipe, plane) + ((4) * (level))) + _MMIO(_PLANE_WM_BASE(pipe, plane) + ((4) * (level))) #define _PLANE_WM_TRANS_1(pipe) \ - _PIPE(pipe, PLANE_WM_TRANS_1_A_0, PLANE_WM_TRANS_1_B_0) + _PIPE(pipe, _PLANE_WM_TRANS_1_A_0, _PLANE_WM_TRANS_1_B_0) #define _PLANE_WM_TRANS_2(pipe) \ - _PIPE(pipe, PLANE_WM_TRANS_2_A_0, PLANE_WM_TRANS_2_B_0) + _PIPE(pipe, _PLANE_WM_TRANS_2_A_0, _PLANE_WM_TRANS_2_B_0) #define PLANE_WM_TRANS(pipe, plane) \ - _PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe)) + _MMIO(_PLANE(plane, _PLANE_WM_TRANS_1(pipe), _PLANE_WM_TRANS_2(pipe))) /* define the Watermark register on Ironlake */ -#define WM0_PIPEA_ILK 0x45100 +#define WM0_PIPEA_ILK _MMIO(0x45100) #define WM0_PIPE_PLANE_MASK (0xffff<<16) #define WM0_PIPE_PLANE_SHIFT 16 #define WM0_PIPE_SPRITE_MASK (0xff<<8) #define WM0_PIPE_SPRITE_SHIFT 8 #define WM0_PIPE_CURSOR_MASK (0xff) -#define WM0_PIPEB_ILK 0x45104 -#define WM0_PIPEC_IVB 0x45200 -#define WM1_LP_ILK 0x45108 +#define WM0_PIPEB_ILK _MMIO(0x45104) +#define WM0_PIPEC_IVB _MMIO(0x45200) +#define WM1_LP_ILK _MMIO(0x45108) #define WM1_LP_SR_EN (1<<31) #define WM1_LP_LATENCY_SHIFT 24 #define WM1_LP_LATENCY_MASK (0x7f<<24) @@ -4633,13 +4855,13 @@ enum skl_disp_power_wells { #define WM1_LP_SR_MASK (0x7ff<<8) #define WM1_LP_SR_SHIFT 8 #define WM1_LP_CURSOR_MASK (0xff) -#define WM2_LP_ILK 0x4510c +#define WM2_LP_ILK _MMIO(0x4510c) #define WM2_LP_EN (1<<31) -#define WM3_LP_ILK 0x45110 +#define WM3_LP_ILK _MMIO(0x45110) #define WM3_LP_EN (1<<31) -#define WM1S_LP_ILK 0x45120 -#define WM2S_LP_IVB 0x45124 -#define WM3S_LP_IVB 0x45128 +#define WM1S_LP_ILK _MMIO(0x45120) +#define WM2S_LP_IVB _MMIO(0x45124) +#define WM3S_LP_IVB _MMIO(0x45128) #define WM1S_LP_EN (1<<31) #define HSW_WM_LP_VAL(lat, fbc, pri, cur) \ @@ -4647,7 +4869,7 @@ enum skl_disp_power_wells { ((fbc) << WM1_LP_FBC_SHIFT) | ((pri) << WM1_LP_SR_SHIFT) | (cur)) /* Memory latency timer register */ -#define MLTR_ILK 0x11222 +#define MLTR_ILK _MMIO(0x11222) #define MLTR_WM1_SHIFT 0 #define MLTR_WM2_SHIFT 8 /* the unit of memory self-refresh latency time is 0.5us */ @@ -4655,7 +4877,7 @@ enum skl_disp_power_wells { /* the address where we get all kinds of latency value */ -#define SSKPD 0x5d10 +#define SSKPD _MMIO(0x5d10) #define SSKPD_WM_MASK 0x3f #define SSKPD_WM0_SHIFT 0 #define SSKPD_WM1_SHIFT 8 @@ -4686,10 +4908,10 @@ enum skl_disp_power_wells { #define PIPE_PIXEL_MASK 0x00ffffff #define PIPE_PIXEL_SHIFT 0 /* GM45+ just has to be different */ -#define _PIPEA_FRMCOUNT_GM45 0x70040 -#define _PIPEA_FLIPCOUNT_GM45 0x70044 -#define PIPE_FRMCOUNT_GM45(pipe) _PIPE2(pipe, _PIPEA_FRMCOUNT_GM45) -#define PIPE_FLIPCOUNT_GM45(pipe) _PIPE2(pipe, _PIPEA_FLIPCOUNT_GM45) +#define _PIPEA_FRMCOUNT_G4X 0x70040 +#define _PIPEA_FLIPCOUNT_G4X 0x70044 +#define PIPE_FRMCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FRMCOUNT_G4X) +#define PIPE_FLIPCOUNT_G4X(pipe) _MMIO_PIPE2(pipe, _PIPEA_FLIPCOUNT_G4X) /* Cursor A & B regs */ #define _CURACNTR 0x70080 @@ -4727,7 +4949,7 @@ enum skl_disp_power_wells { #define CURSOR_POS_SIGN 0x8000 #define CURSOR_X_SHIFT 0 #define CURSOR_Y_SHIFT 16 -#define CURSIZE 0x700a0 +#define CURSIZE _MMIO(0x700a0) #define _CURBCNTR 0x700c0 #define _CURBBASE 0x700c4 #define _CURBPOS 0x700c8 @@ -4736,7 +4958,7 @@ enum skl_disp_power_wells { #define _CURBBASE_IVB 0x71084 #define _CURBPOS_IVB 0x71088 -#define _CURSOR2(pipe, reg) (dev_priv->info.cursor_offsets[(pipe)] - \ +#define _CURSOR2(pipe, reg) _MMIO(dev_priv->info.cursor_offsets[(pipe)] - \ dev_priv->info.cursor_offsets[PIPE_A] + (reg) + \ dev_priv->info.display_mmio_offset) @@ -4797,16 +5019,16 @@ enum skl_disp_power_wells { #define _DSPAOFFSET 0x701A4 /* HSW */ #define _DSPASURFLIVE 0x701AC -#define DSPCNTR(plane) _PIPE2(plane, _DSPACNTR) -#define DSPADDR(plane) _PIPE2(plane, _DSPAADDR) -#define DSPSTRIDE(plane) _PIPE2(plane, _DSPASTRIDE) -#define DSPPOS(plane) _PIPE2(plane, _DSPAPOS) -#define DSPSIZE(plane) _PIPE2(plane, _DSPASIZE) -#define DSPSURF(plane) _PIPE2(plane, _DSPASURF) -#define DSPTILEOFF(plane) _PIPE2(plane, _DSPATILEOFF) -#define DSPLINOFF(plane) DSPADDR(plane) -#define DSPOFFSET(plane) _PIPE2(plane, _DSPAOFFSET) -#define DSPSURFLIVE(plane) _PIPE2(plane, _DSPASURFLIVE) +#define DSPCNTR(plane) _MMIO_PIPE2(plane, _DSPACNTR) +#define DSPADDR(plane) _MMIO_PIPE2(plane, _DSPAADDR) +#define DSPSTRIDE(plane) _MMIO_PIPE2(plane, _DSPASTRIDE) +#define DSPPOS(plane) _MMIO_PIPE2(plane, _DSPAPOS) +#define DSPSIZE(plane) _MMIO_PIPE2(plane, _DSPASIZE) +#define DSPSURF(plane) _MMIO_PIPE2(plane, _DSPASURF) +#define DSPTILEOFF(plane) _MMIO_PIPE2(plane, _DSPATILEOFF) +#define DSPLINOFF(plane) DSPADDR(plane) +#define DSPOFFSET(plane) _MMIO_PIPE2(plane, _DSPAOFFSET) +#define DSPSURFLIVE(plane) _MMIO_PIPE2(plane, _DSPASURFLIVE) /* CHV pipe B blender and primary plane */ #define _CHV_BLEND_A 0x60a00 @@ -4820,31 +5042,32 @@ enum skl_disp_power_wells { #define _PRIMCNSTALPHA_A 0x60a10 #define PRIM_CONST_ALPHA_ENABLE (1<<31) -#define CHV_BLEND(pipe) _TRANSCODER2(pipe, _CHV_BLEND_A) -#define CHV_CANVAS(pipe) _TRANSCODER2(pipe, _CHV_CANVAS_A) -#define PRIMPOS(plane) _TRANSCODER2(plane, _PRIMPOS_A) -#define PRIMSIZE(plane) _TRANSCODER2(plane, _PRIMSIZE_A) -#define PRIMCNSTALPHA(plane) _TRANSCODER2(plane, _PRIMCNSTALPHA_A) +#define CHV_BLEND(pipe) _MMIO_TRANS2(pipe, _CHV_BLEND_A) +#define CHV_CANVAS(pipe) _MMIO_TRANS2(pipe, _CHV_CANVAS_A) +#define PRIMPOS(plane) _MMIO_TRANS2(plane, _PRIMPOS_A) +#define PRIMSIZE(plane) _MMIO_TRANS2(plane, _PRIMSIZE_A) +#define PRIMCNSTALPHA(plane) _MMIO_TRANS2(plane, _PRIMCNSTALPHA_A) /* Display/Sprite base address macros */ #define DISP_BASEADDR_MASK (0xfffff000) #define I915_LO_DISPBASE(val) (val & ~DISP_BASEADDR_MASK) #define I915_HI_DISPBASE(val) (val & DISP_BASEADDR_MASK) -/* VBIOS flags */ -#define SWF00 (dev_priv->info.display_mmio_offset + 0x71410) -#define SWF01 (dev_priv->info.display_mmio_offset + 0x71414) -#define SWF02 (dev_priv->info.display_mmio_offset + 0x71418) -#define SWF03 (dev_priv->info.display_mmio_offset + 0x7141c) -#define SWF04 (dev_priv->info.display_mmio_offset + 0x71420) -#define SWF05 (dev_priv->info.display_mmio_offset + 0x71424) -#define SWF06 (dev_priv->info.display_mmio_offset + 0x71428) -#define SWF10 (dev_priv->info.display_mmio_offset + 0x70410) -#define SWF11 (dev_priv->info.display_mmio_offset + 0x70414) -#define SWF14 (dev_priv->info.display_mmio_offset + 0x71420) -#define SWF30 (dev_priv->info.display_mmio_offset + 0x72414) -#define SWF31 (dev_priv->info.display_mmio_offset + 0x72418) -#define SWF32 (dev_priv->info.display_mmio_offset + 0x7241c) +/* + * VBIOS flags + * gen2: + * [00:06] alm,mgm + * [10:16] all + * [30:32] alm,mgm + * gen3+: + * [00:0f] all + * [10:1f] all + * [30:32] all + */ +#define SWF0(i) _MMIO(dev_priv->info.display_mmio_offset + 0x70410 + (i) * 4) +#define SWF1(i) _MMIO(dev_priv->info.display_mmio_offset + 0x71410 + (i) * 4) +#define SWF3(i) _MMIO(dev_priv->info.display_mmio_offset + 0x72414 + (i) * 4) +#define SWF_ILK(i) _MMIO(0x4F000 + (i) * 4) /* Pipe B */ #define _PIPEBDSL (dev_priv->info.display_mmio_offset + 0x71000) @@ -4852,8 +5075,8 @@ enum skl_disp_power_wells { #define _PIPEBSTAT (dev_priv->info.display_mmio_offset + 0x71024) #define _PIPEBFRAMEHIGH 0x71040 #define _PIPEBFRAMEPIXEL 0x71044 -#define _PIPEB_FRMCOUNT_GM45 (dev_priv->info.display_mmio_offset + 0x71040) -#define _PIPEB_FLIPCOUNT_GM45 (dev_priv->info.display_mmio_offset + 0x71044) +#define _PIPEB_FRMCOUNT_G4X (dev_priv->info.display_mmio_offset + 0x71040) +#define _PIPEB_FLIPCOUNT_G4X (dev_priv->info.display_mmio_offset + 0x71044) /* Display B control */ @@ -4926,18 +5149,18 @@ enum skl_disp_power_wells { #define _DVSBSCALE 0x73204 #define _DVSBGAMC 0x73300 -#define DVSCNTR(pipe) _PIPE(pipe, _DVSACNTR, _DVSBCNTR) -#define DVSLINOFF(pipe) _PIPE(pipe, _DVSALINOFF, _DVSBLINOFF) -#define DVSSTRIDE(pipe) _PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE) -#define DVSPOS(pipe) _PIPE(pipe, _DVSAPOS, _DVSBPOS) -#define DVSSURF(pipe) _PIPE(pipe, _DVSASURF, _DVSBSURF) -#define DVSKEYMAX(pipe) _PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL) -#define DVSSIZE(pipe) _PIPE(pipe, _DVSASIZE, _DVSBSIZE) -#define DVSSCALE(pipe) _PIPE(pipe, _DVSASCALE, _DVSBSCALE) -#define DVSTILEOFF(pipe) _PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF) -#define DVSKEYVAL(pipe) _PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL) -#define DVSKEYMSK(pipe) _PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK) -#define DVSSURFLIVE(pipe) _PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE) +#define DVSCNTR(pipe) _MMIO_PIPE(pipe, _DVSACNTR, _DVSBCNTR) +#define DVSLINOFF(pipe) _MMIO_PIPE(pipe, _DVSALINOFF, _DVSBLINOFF) +#define DVSSTRIDE(pipe) _MMIO_PIPE(pipe, _DVSASTRIDE, _DVSBSTRIDE) +#define DVSPOS(pipe) _MMIO_PIPE(pipe, _DVSAPOS, _DVSBPOS) +#define DVSSURF(pipe) _MMIO_PIPE(pipe, _DVSASURF, _DVSBSURF) +#define DVSKEYMAX(pipe) _MMIO_PIPE(pipe, _DVSAKEYMAXVAL, _DVSBKEYMAXVAL) +#define DVSSIZE(pipe) _MMIO_PIPE(pipe, _DVSASIZE, _DVSBSIZE) +#define DVSSCALE(pipe) _MMIO_PIPE(pipe, _DVSASCALE, _DVSBSCALE) +#define DVSTILEOFF(pipe) _MMIO_PIPE(pipe, _DVSATILEOFF, _DVSBTILEOFF) +#define DVSKEYVAL(pipe) _MMIO_PIPE(pipe, _DVSAKEYVAL, _DVSBKEYVAL) +#define DVSKEYMSK(pipe) _MMIO_PIPE(pipe, _DVSAKEYMSK, _DVSBKEYMSK) +#define DVSSURFLIVE(pipe) _MMIO_PIPE(pipe, _DVSASURFLIVE, _DVSBSURFLIVE) #define _SPRA_CTL 0x70280 #define SPRITE_ENABLE (1<<31) @@ -5000,20 +5223,20 @@ enum skl_disp_power_wells { #define _SPRB_SCALE 0x71304 #define _SPRB_GAMC 0x71400 -#define SPRCTL(pipe) _PIPE(pipe, _SPRA_CTL, _SPRB_CTL) -#define SPRLINOFF(pipe) _PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF) -#define SPRSTRIDE(pipe) _PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE) -#define SPRPOS(pipe) _PIPE(pipe, _SPRA_POS, _SPRB_POS) -#define SPRSIZE(pipe) _PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE) -#define SPRKEYVAL(pipe) _PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL) -#define SPRKEYMSK(pipe) _PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK) -#define SPRSURF(pipe) _PIPE(pipe, _SPRA_SURF, _SPRB_SURF) -#define SPRKEYMAX(pipe) _PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX) -#define SPRTILEOFF(pipe) _PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF) -#define SPROFFSET(pipe) _PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET) -#define SPRSCALE(pipe) _PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE) -#define SPRGAMC(pipe) _PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC) -#define SPRSURFLIVE(pipe) _PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE) +#define SPRCTL(pipe) _MMIO_PIPE(pipe, _SPRA_CTL, _SPRB_CTL) +#define SPRLINOFF(pipe) _MMIO_PIPE(pipe, _SPRA_LINOFF, _SPRB_LINOFF) +#define SPRSTRIDE(pipe) _MMIO_PIPE(pipe, _SPRA_STRIDE, _SPRB_STRIDE) +#define SPRPOS(pipe) _MMIO_PIPE(pipe, _SPRA_POS, _SPRB_POS) +#define SPRSIZE(pipe) _MMIO_PIPE(pipe, _SPRA_SIZE, _SPRB_SIZE) +#define SPRKEYVAL(pipe) _MMIO_PIPE(pipe, _SPRA_KEYVAL, _SPRB_KEYVAL) +#define SPRKEYMSK(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMSK, _SPRB_KEYMSK) +#define SPRSURF(pipe) _MMIO_PIPE(pipe, _SPRA_SURF, _SPRB_SURF) +#define SPRKEYMAX(pipe) _MMIO_PIPE(pipe, _SPRA_KEYMAX, _SPRB_KEYMAX) +#define SPRTILEOFF(pipe) _MMIO_PIPE(pipe, _SPRA_TILEOFF, _SPRB_TILEOFF) +#define SPROFFSET(pipe) _MMIO_PIPE(pipe, _SPRA_OFFSET, _SPRB_OFFSET) +#define SPRSCALE(pipe) _MMIO_PIPE(pipe, _SPRA_SCALE, _SPRB_SCALE) +#define SPRGAMC(pipe) _MMIO_PIPE(pipe, _SPRA_GAMC, _SPRB_GAMC) +#define SPRSURFLIVE(pipe) _MMIO_PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE) #define _SPACNTR (VLV_DISPLAY_BASE + 0x72180) #define SP_ENABLE (1<<31) @@ -5063,18 +5286,18 @@ enum skl_disp_power_wells { #define _SPBCONSTALPHA (VLV_DISPLAY_BASE + 0x722a8) #define _SPBGAMC (VLV_DISPLAY_BASE + 0x722f4) -#define SPCNTR(pipe, plane) _PIPE(pipe * 2 + plane, _SPACNTR, _SPBCNTR) -#define SPLINOFF(pipe, plane) _PIPE(pipe * 2 + plane, _SPALINOFF, _SPBLINOFF) -#define SPSTRIDE(pipe, plane) _PIPE(pipe * 2 + plane, _SPASTRIDE, _SPBSTRIDE) -#define SPPOS(pipe, plane) _PIPE(pipe * 2 + plane, _SPAPOS, _SPBPOS) -#define SPSIZE(pipe, plane) _PIPE(pipe * 2 + plane, _SPASIZE, _SPBSIZE) -#define SPKEYMINVAL(pipe, plane) _PIPE(pipe * 2 + plane, _SPAKEYMINVAL, _SPBKEYMINVAL) -#define SPKEYMSK(pipe, plane) _PIPE(pipe * 2 + plane, _SPAKEYMSK, _SPBKEYMSK) -#define SPSURF(pipe, plane) _PIPE(pipe * 2 + plane, _SPASURF, _SPBSURF) -#define SPKEYMAXVAL(pipe, plane) _PIPE(pipe * 2 + plane, _SPAKEYMAXVAL, _SPBKEYMAXVAL) -#define SPTILEOFF(pipe, plane) _PIPE(pipe * 2 + plane, _SPATILEOFF, _SPBTILEOFF) -#define SPCONSTALPHA(pipe, plane) _PIPE(pipe * 2 + plane, _SPACONSTALPHA, _SPBCONSTALPHA) -#define SPGAMC(pipe, plane) _PIPE(pipe * 2 + plane, _SPAGAMC, _SPBGAMC) +#define SPCNTR(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPACNTR, _SPBCNTR) +#define SPLINOFF(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPALINOFF, _SPBLINOFF) +#define SPSTRIDE(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPASTRIDE, _SPBSTRIDE) +#define SPPOS(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAPOS, _SPBPOS) +#define SPSIZE(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPASIZE, _SPBSIZE) +#define SPKEYMINVAL(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAKEYMINVAL, _SPBKEYMINVAL) +#define SPKEYMSK(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAKEYMSK, _SPBKEYMSK) +#define SPSURF(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPASURF, _SPBSURF) +#define SPKEYMAXVAL(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAKEYMAXVAL, _SPBKEYMAXVAL) +#define SPTILEOFF(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPATILEOFF, _SPBTILEOFF) +#define SPCONSTALPHA(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPACONSTALPHA, _SPBCONSTALPHA) +#define SPGAMC(pipe, plane) _MMIO_PIPE((pipe) * 2 + (plane), _SPAGAMC, _SPBGAMC) /* * CHV pipe B sprite CSC @@ -5083,29 +5306,29 @@ enum skl_disp_power_wells { * |yg| = |c3 c4 c5| x |yg + yg_ioff| + |yg_ooff| * |cb| |c6 c7 c8| |cb + cr_ioff| |cb_ooff| */ -#define SPCSCYGOFF(sprite) (VLV_DISPLAY_BASE + 0x6d900 + (sprite) * 0x1000) -#define SPCSCCBOFF(sprite) (VLV_DISPLAY_BASE + 0x6d904 + (sprite) * 0x1000) -#define SPCSCCROFF(sprite) (VLV_DISPLAY_BASE + 0x6d908 + (sprite) * 0x1000) +#define SPCSCYGOFF(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d900 + (sprite) * 0x1000) +#define SPCSCCBOFF(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d904 + (sprite) * 0x1000) +#define SPCSCCROFF(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d908 + (sprite) * 0x1000) #define SPCSC_OOFF(x) (((x) & 0x7ff) << 16) /* s11 */ #define SPCSC_IOFF(x) (((x) & 0x7ff) << 0) /* s11 */ -#define SPCSCC01(sprite) (VLV_DISPLAY_BASE + 0x6d90c + (sprite) * 0x1000) -#define SPCSCC23(sprite) (VLV_DISPLAY_BASE + 0x6d910 + (sprite) * 0x1000) -#define SPCSCC45(sprite) (VLV_DISPLAY_BASE + 0x6d914 + (sprite) * 0x1000) -#define SPCSCC67(sprite) (VLV_DISPLAY_BASE + 0x6d918 + (sprite) * 0x1000) -#define SPCSCC8(sprite) (VLV_DISPLAY_BASE + 0x6d91c + (sprite) * 0x1000) +#define SPCSCC01(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d90c + (sprite) * 0x1000) +#define SPCSCC23(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d910 + (sprite) * 0x1000) +#define SPCSCC45(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d914 + (sprite) * 0x1000) +#define SPCSCC67(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d918 + (sprite) * 0x1000) +#define SPCSCC8(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d91c + (sprite) * 0x1000) #define SPCSC_C1(x) (((x) & 0x7fff) << 16) /* s3.12 */ #define SPCSC_C0(x) (((x) & 0x7fff) << 0) /* s3.12 */ -#define SPCSCYGICLAMP(sprite) (VLV_DISPLAY_BASE + 0x6d920 + (sprite) * 0x1000) -#define SPCSCCBICLAMP(sprite) (VLV_DISPLAY_BASE + 0x6d924 + (sprite) * 0x1000) -#define SPCSCCRICLAMP(sprite) (VLV_DISPLAY_BASE + 0x6d928 + (sprite) * 0x1000) +#define SPCSCYGICLAMP(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d920 + (sprite) * 0x1000) +#define SPCSCCBICLAMP(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d924 + (sprite) * 0x1000) +#define SPCSCCRICLAMP(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d928 + (sprite) * 0x1000) #define SPCSC_IMAX(x) (((x) & 0x7ff) << 16) /* s11 */ #define SPCSC_IMIN(x) (((x) & 0x7ff) << 0) /* s11 */ -#define SPCSCYGOCLAMP(sprite) (VLV_DISPLAY_BASE + 0x6d92c + (sprite) * 0x1000) -#define SPCSCCBOCLAMP(sprite) (VLV_DISPLAY_BASE + 0x6d930 + (sprite) * 0x1000) -#define SPCSCCROCLAMP(sprite) (VLV_DISPLAY_BASE + 0x6d934 + (sprite) * 0x1000) +#define SPCSCYGOCLAMP(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d92c + (sprite) * 0x1000) +#define SPCSCCBOCLAMP(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d930 + (sprite) * 0x1000) +#define SPCSCCROCLAMP(sprite) _MMIO(VLV_DISPLAY_BASE + 0x6d934 + (sprite) * 0x1000) #define SPCSC_OMAX(x) ((x) << 16) /* u10 */ #define SPCSC_OMIN(x) ((x) << 0) /* u10 */ @@ -5186,7 +5409,7 @@ enum skl_disp_power_wells { #define _PLANE_CTL_2(pipe) _PIPE(pipe, _PLANE_CTL_2_A, _PLANE_CTL_2_B) #define _PLANE_CTL_3(pipe) _PIPE(pipe, _PLANE_CTL_3_A, _PLANE_CTL_3_B) #define PLANE_CTL(pipe, plane) \ - _PLANE(plane, _PLANE_CTL_1(pipe), _PLANE_CTL_2(pipe)) + _MMIO_PLANE(plane, _PLANE_CTL_1(pipe), _PLANE_CTL_2(pipe)) #define _PLANE_STRIDE_1_B 0x71188 #define _PLANE_STRIDE_2_B 0x71288 @@ -5198,7 +5421,7 @@ enum skl_disp_power_wells { #define _PLANE_STRIDE_3(pipe) \ _PIPE(pipe, _PLANE_STRIDE_3_A, _PLANE_STRIDE_3_B) #define PLANE_STRIDE(pipe, plane) \ - _PLANE(plane, _PLANE_STRIDE_1(pipe), _PLANE_STRIDE_2(pipe)) + _MMIO_PLANE(plane, _PLANE_STRIDE_1(pipe), _PLANE_STRIDE_2(pipe)) #define _PLANE_POS_1_B 0x7118c #define _PLANE_POS_2_B 0x7128c @@ -5207,7 +5430,7 @@ enum skl_disp_power_wells { #define _PLANE_POS_2(pipe) _PIPE(pipe, _PLANE_POS_2_A, _PLANE_POS_2_B) #define _PLANE_POS_3(pipe) _PIPE(pipe, _PLANE_POS_3_A, _PLANE_POS_3_B) #define PLANE_POS(pipe, plane) \ - _PLANE(plane, _PLANE_POS_1(pipe), _PLANE_POS_2(pipe)) + _MMIO_PLANE(plane, _PLANE_POS_1(pipe), _PLANE_POS_2(pipe)) #define _PLANE_SIZE_1_B 0x71190 #define _PLANE_SIZE_2_B 0x71290 @@ -5216,7 +5439,7 @@ enum skl_disp_power_wells { #define _PLANE_SIZE_2(pipe) _PIPE(pipe, _PLANE_SIZE_2_A, _PLANE_SIZE_2_B) #define _PLANE_SIZE_3(pipe) _PIPE(pipe, _PLANE_SIZE_3_A, _PLANE_SIZE_3_B) #define PLANE_SIZE(pipe, plane) \ - _PLANE(plane, _PLANE_SIZE_1(pipe), _PLANE_SIZE_2(pipe)) + _MMIO_PLANE(plane, _PLANE_SIZE_1(pipe), _PLANE_SIZE_2(pipe)) #define _PLANE_SURF_1_B 0x7119c #define _PLANE_SURF_2_B 0x7129c @@ -5225,35 +5448,35 @@ enum skl_disp_power_wells { #define _PLANE_SURF_2(pipe) _PIPE(pipe, _PLANE_SURF_2_A, _PLANE_SURF_2_B) #define _PLANE_SURF_3(pipe) _PIPE(pipe, _PLANE_SURF_3_A, _PLANE_SURF_3_B) #define PLANE_SURF(pipe, plane) \ - _PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe)) + _MMIO_PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe)) #define _PLANE_OFFSET_1_B 0x711a4 #define _PLANE_OFFSET_2_B 0x712a4 #define _PLANE_OFFSET_1(pipe) _PIPE(pipe, _PLANE_OFFSET_1_A, _PLANE_OFFSET_1_B) #define _PLANE_OFFSET_2(pipe) _PIPE(pipe, _PLANE_OFFSET_2_A, _PLANE_OFFSET_2_B) #define PLANE_OFFSET(pipe, plane) \ - _PLANE(plane, _PLANE_OFFSET_1(pipe), _PLANE_OFFSET_2(pipe)) + _MMIO_PLANE(plane, _PLANE_OFFSET_1(pipe), _PLANE_OFFSET_2(pipe)) #define _PLANE_KEYVAL_1_B 0x71194 #define _PLANE_KEYVAL_2_B 0x71294 #define _PLANE_KEYVAL_1(pipe) _PIPE(pipe, _PLANE_KEYVAL_1_A, _PLANE_KEYVAL_1_B) #define _PLANE_KEYVAL_2(pipe) _PIPE(pipe, _PLANE_KEYVAL_2_A, _PLANE_KEYVAL_2_B) #define PLANE_KEYVAL(pipe, plane) \ - _PLANE(plane, _PLANE_KEYVAL_1(pipe), _PLANE_KEYVAL_2(pipe)) + _MMIO_PLANE(plane, _PLANE_KEYVAL_1(pipe), _PLANE_KEYVAL_2(pipe)) #define _PLANE_KEYMSK_1_B 0x71198 #define _PLANE_KEYMSK_2_B 0x71298 #define _PLANE_KEYMSK_1(pipe) _PIPE(pipe, _PLANE_KEYMSK_1_A, _PLANE_KEYMSK_1_B) #define _PLANE_KEYMSK_2(pipe) _PIPE(pipe, _PLANE_KEYMSK_2_A, _PLANE_KEYMSK_2_B) #define PLANE_KEYMSK(pipe, plane) \ - _PLANE(plane, _PLANE_KEYMSK_1(pipe), _PLANE_KEYMSK_2(pipe)) + _MMIO_PLANE(plane, _PLANE_KEYMSK_1(pipe), _PLANE_KEYMSK_2(pipe)) #define _PLANE_KEYMAX_1_B 0x711a0 #define _PLANE_KEYMAX_2_B 0x712a0 #define _PLANE_KEYMAX_1(pipe) _PIPE(pipe, _PLANE_KEYMAX_1_A, _PLANE_KEYMAX_1_B) #define _PLANE_KEYMAX_2(pipe) _PIPE(pipe, _PLANE_KEYMAX_2_A, _PLANE_KEYMAX_2_B) #define PLANE_KEYMAX(pipe, plane) \ - _PLANE(plane, _PLANE_KEYMAX_1(pipe), _PLANE_KEYMAX_2(pipe)) + _MMIO_PLANE(plane, _PLANE_KEYMAX_1(pipe), _PLANE_KEYMAX_2(pipe)) #define _PLANE_BUF_CFG_1_B 0x7127c #define _PLANE_BUF_CFG_2_B 0x7137c @@ -5262,7 +5485,7 @@ enum skl_disp_power_wells { #define _PLANE_BUF_CFG_2(pipe) \ _PIPE(pipe, _PLANE_BUF_CFG_2_A, _PLANE_BUF_CFG_2_B) #define PLANE_BUF_CFG(pipe, plane) \ - _PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe)) + _MMIO_PLANE(plane, _PLANE_BUF_CFG_1(pipe), _PLANE_BUF_CFG_2(pipe)) #define _PLANE_NV12_BUF_CFG_1_B 0x71278 #define _PLANE_NV12_BUF_CFG_2_B 0x71378 @@ -5271,56 +5494,58 @@ enum skl_disp_power_wells { #define _PLANE_NV12_BUF_CFG_2(pipe) \ _PIPE(pipe, _PLANE_NV12_BUF_CFG_2_A, _PLANE_NV12_BUF_CFG_2_B) #define PLANE_NV12_BUF_CFG(pipe, plane) \ - _PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe)) + _MMIO_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe)) /* SKL new cursor registers */ #define _CUR_BUF_CFG_A 0x7017c #define _CUR_BUF_CFG_B 0x7117c -#define CUR_BUF_CFG(pipe) _PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B) +#define CUR_BUF_CFG(pipe) _MMIO_PIPE(pipe, _CUR_BUF_CFG_A, _CUR_BUF_CFG_B) /* VBIOS regs */ -#define VGACNTRL 0x71400 +#define VGACNTRL _MMIO(0x71400) # define VGA_DISP_DISABLE (1 << 31) # define VGA_2X_MODE (1 << 30) # define VGA_PIPE_B_SELECT (1 << 29) -#define VLV_VGACNTRL (VLV_DISPLAY_BASE + 0x71400) +#define VLV_VGACNTRL _MMIO(VLV_DISPLAY_BASE + 0x71400) /* Ironlake */ -#define CPU_VGACNTRL 0x41000 +#define CPU_VGACNTRL _MMIO(0x41000) -#define DIGITAL_PORT_HOTPLUG_CNTRL 0x44030 -#define DIGITAL_PORTA_HOTPLUG_ENABLE (1 << 4) -#define DIGITAL_PORTA_SHORT_PULSE_2MS (0 << 2) -#define DIGITAL_PORTA_SHORT_PULSE_4_5MS (1 << 2) -#define DIGITAL_PORTA_SHORT_PULSE_6MS (2 << 2) -#define DIGITAL_PORTA_SHORT_PULSE_100MS (3 << 2) -#define DIGITAL_PORTA_NO_DETECT (0 << 0) -#define DIGITAL_PORTA_LONG_PULSE_DETECT_MASK (1 << 1) -#define DIGITAL_PORTA_SHORT_PULSE_DETECT_MASK (1 << 0) +#define DIGITAL_PORT_HOTPLUG_CNTRL _MMIO(0x44030) +#define DIGITAL_PORTA_HOTPLUG_ENABLE (1 << 4) +#define DIGITAL_PORTA_PULSE_DURATION_2ms (0 << 2) /* pre-HSW */ +#define DIGITAL_PORTA_PULSE_DURATION_4_5ms (1 << 2) /* pre-HSW */ +#define DIGITAL_PORTA_PULSE_DURATION_6ms (2 << 2) /* pre-HSW */ +#define DIGITAL_PORTA_PULSE_DURATION_100ms (3 << 2) /* pre-HSW */ +#define DIGITAL_PORTA_PULSE_DURATION_MASK (3 << 2) /* pre-HSW */ +#define DIGITAL_PORTA_HOTPLUG_STATUS_MASK (3 << 0) +#define DIGITAL_PORTA_HOTPLUG_NO_DETECT (0 << 0) +#define DIGITAL_PORTA_HOTPLUG_SHORT_DETECT (1 << 0) +#define DIGITAL_PORTA_HOTPLUG_LONG_DETECT (2 << 0) /* refresh rate hardware control */ -#define RR_HW_CTL 0x45300 +#define RR_HW_CTL _MMIO(0x45300) #define RR_HW_LOW_POWER_FRAMES_MASK 0xff #define RR_HW_HIGH_POWER_FRAMES_MASK 0xff00 -#define FDI_PLL_BIOS_0 0x46000 +#define FDI_PLL_BIOS_0 _MMIO(0x46000) #define FDI_PLL_FB_CLOCK_MASK 0xff -#define FDI_PLL_BIOS_1 0x46004 -#define FDI_PLL_BIOS_2 0x46008 -#define DISPLAY_PORT_PLL_BIOS_0 0x4600c -#define DISPLAY_PORT_PLL_BIOS_1 0x46010 -#define DISPLAY_PORT_PLL_BIOS_2 0x46014 +#define FDI_PLL_BIOS_1 _MMIO(0x46004) +#define FDI_PLL_BIOS_2 _MMIO(0x46008) +#define DISPLAY_PORT_PLL_BIOS_0 _MMIO(0x4600c) +#define DISPLAY_PORT_PLL_BIOS_1 _MMIO(0x46010) +#define DISPLAY_PORT_PLL_BIOS_2 _MMIO(0x46014) -#define PCH_3DCGDIS0 0x46020 +#define PCH_3DCGDIS0 _MMIO(0x46020) # define MARIUNIT_CLOCK_GATE_DISABLE (1 << 18) # define SVSMUNIT_CLOCK_GATE_DISABLE (1 << 1) -#define PCH_3DCGDIS1 0x46024 +#define PCH_3DCGDIS1 _MMIO(0x46024) # define VFMUNIT_CLOCK_GATE_DISABLE (1 << 11) -#define FDI_PLL_FREQ_CTL 0x46030 +#define FDI_PLL_FREQ_CTL _MMIO(0x46030) #define FDI_PLL_FREQ_CHANGE_REQUEST (1<<24) #define FDI_PLL_FREQ_LOCK_LIMIT_MASK 0xfff00 #define FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK 0xff @@ -5357,14 +5582,14 @@ enum skl_disp_power_wells { #define _PIPEB_LINK_M2 0x61048 #define _PIPEB_LINK_N2 0x6104c -#define PIPE_DATA_M1(tran) _TRANSCODER2(tran, _PIPEA_DATA_M1) -#define PIPE_DATA_N1(tran) _TRANSCODER2(tran, _PIPEA_DATA_N1) -#define PIPE_DATA_M2(tran) _TRANSCODER2(tran, _PIPEA_DATA_M2) -#define PIPE_DATA_N2(tran) _TRANSCODER2(tran, _PIPEA_DATA_N2) -#define PIPE_LINK_M1(tran) _TRANSCODER2(tran, _PIPEA_LINK_M1) -#define PIPE_LINK_N1(tran) _TRANSCODER2(tran, _PIPEA_LINK_N1) -#define PIPE_LINK_M2(tran) _TRANSCODER2(tran, _PIPEA_LINK_M2) -#define PIPE_LINK_N2(tran) _TRANSCODER2(tran, _PIPEA_LINK_N2) +#define PIPE_DATA_M1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M1) +#define PIPE_DATA_N1(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N1) +#define PIPE_DATA_M2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_M2) +#define PIPE_DATA_N2(tran) _MMIO_TRANS2(tran, _PIPEA_DATA_N2) +#define PIPE_LINK_M1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M1) +#define PIPE_LINK_N1(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N1) +#define PIPE_LINK_M2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_M2) +#define PIPE_LINK_N2(tran) _MMIO_TRANS2(tran, _PIPEA_LINK_N2) /* CPU panel fitter */ /* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */ @@ -5387,11 +5612,11 @@ enum skl_disp_power_wells { #define _PFA_HSCALE 0x68090 #define _PFB_HSCALE 0x68890 -#define PF_CTL(pipe) _PIPE(pipe, _PFA_CTL_1, _PFB_CTL_1) -#define PF_WIN_SZ(pipe) _PIPE(pipe, _PFA_WIN_SZ, _PFB_WIN_SZ) -#define PF_WIN_POS(pipe) _PIPE(pipe, _PFA_WIN_POS, _PFB_WIN_POS) -#define PF_VSCALE(pipe) _PIPE(pipe, _PFA_VSCALE, _PFB_VSCALE) -#define PF_HSCALE(pipe) _PIPE(pipe, _PFA_HSCALE, _PFB_HSCALE) +#define PF_CTL(pipe) _MMIO_PIPE(pipe, _PFA_CTL_1, _PFB_CTL_1) +#define PF_WIN_SZ(pipe) _MMIO_PIPE(pipe, _PFA_WIN_SZ, _PFB_WIN_SZ) +#define PF_WIN_POS(pipe) _MMIO_PIPE(pipe, _PFA_WIN_POS, _PFB_WIN_POS) +#define PF_VSCALE(pipe) _MMIO_PIPE(pipe, _PFA_VSCALE, _PFB_VSCALE) +#define PF_HSCALE(pipe) _MMIO_PIPE(pipe, _PFA_HSCALE, _PFB_HSCALE) #define _PSA_CTL 0x68180 #define _PSB_CTL 0x68980 @@ -5401,9 +5626,9 @@ enum skl_disp_power_wells { #define _PSA_WIN_POS 0x68170 #define _PSB_WIN_POS 0x68970 -#define PS_CTL(pipe) _PIPE(pipe, _PSA_CTL, _PSB_CTL) -#define PS_WIN_SZ(pipe) _PIPE(pipe, _PSA_WIN_SZ, _PSB_WIN_SZ) -#define PS_WIN_POS(pipe) _PIPE(pipe, _PSA_WIN_POS, _PSB_WIN_POS) +#define PS_CTL(pipe) _MMIO_PIPE(pipe, _PSA_CTL, _PSB_CTL) +#define PS_WIN_SZ(pipe) _MMIO_PIPE(pipe, _PSA_WIN_SZ, _PSB_WIN_SZ) +#define PS_WIN_POS(pipe) _MMIO_PIPE(pipe, _PSA_WIN_POS, _PSB_WIN_POS) /* * Skylake scalers @@ -5418,7 +5643,7 @@ enum skl_disp_power_wells { #define PS_SCALER_MODE_DYN (0 << 28) #define PS_SCALER_MODE_HQ (1 << 28) #define PS_PLANE_SEL_MASK (7 << 25) -#define PS_PLANE_SEL(plane) ((plane + 1) << 25) +#define PS_PLANE_SEL(plane) (((plane) + 1) << 25) #define PS_FILTER_MASK (3 << 23) #define PS_FILTER_MEDIUM (0 << 23) #define PS_FILTER_EDGE_ENHANCE (2 << 23) @@ -5492,48 +5717,63 @@ enum skl_disp_power_wells { #define _PS_ECC_STAT_1C 0x691D0 #define _ID(id, a, b) ((a) + (id)*((b)-(a))) -#define SKL_PS_CTRL(pipe, id) _PIPE(pipe, \ +#define SKL_PS_CTRL(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_1A_CTRL, _PS_2A_CTRL), \ _ID(id, _PS_1B_CTRL, _PS_2B_CTRL)) -#define SKL_PS_PWR_GATE(pipe, id) _PIPE(pipe, \ +#define SKL_PS_PWR_GATE(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_PWR_GATE_1A, _PS_PWR_GATE_2A), \ _ID(id, _PS_PWR_GATE_1B, _PS_PWR_GATE_2B)) -#define SKL_PS_WIN_POS(pipe, id) _PIPE(pipe, \ +#define SKL_PS_WIN_POS(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_WIN_POS_1A, _PS_WIN_POS_2A), \ _ID(id, _PS_WIN_POS_1B, _PS_WIN_POS_2B)) -#define SKL_PS_WIN_SZ(pipe, id) _PIPE(pipe, \ +#define SKL_PS_WIN_SZ(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_WIN_SZ_1A, _PS_WIN_SZ_2A), \ _ID(id, _PS_WIN_SZ_1B, _PS_WIN_SZ_2B)) -#define SKL_PS_VSCALE(pipe, id) _PIPE(pipe, \ +#define SKL_PS_VSCALE(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_VSCALE_1A, _PS_VSCALE_2A), \ _ID(id, _PS_VSCALE_1B, _PS_VSCALE_2B)) -#define SKL_PS_HSCALE(pipe, id) _PIPE(pipe, \ +#define SKL_PS_HSCALE(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_HSCALE_1A, _PS_HSCALE_2A), \ _ID(id, _PS_HSCALE_1B, _PS_HSCALE_2B)) -#define SKL_PS_VPHASE(pipe, id) _PIPE(pipe, \ +#define SKL_PS_VPHASE(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_VPHASE_1A, _PS_VPHASE_2A), \ _ID(id, _PS_VPHASE_1B, _PS_VPHASE_2B)) -#define SKL_PS_HPHASE(pipe, id) _PIPE(pipe, \ +#define SKL_PS_HPHASE(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_HPHASE_1A, _PS_HPHASE_2A), \ _ID(id, _PS_HPHASE_1B, _PS_HPHASE_2B)) -#define SKL_PS_ECC_STAT(pipe, id) _PIPE(pipe, \ +#define SKL_PS_ECC_STAT(pipe, id) _MMIO_PIPE(pipe, \ _ID(id, _PS_ECC_STAT_1A, _PS_ECC_STAT_2A), \ - _ID(id, _PS_ECC_STAT_1B, _PS_ECC_STAT_2B) + _ID(id, _PS_ECC_STAT_1B, _PS_ECC_STAT_2B)) /* legacy palette */ #define _LGC_PALETTE_A 0x4a000 #define _LGC_PALETTE_B 0x4a800 -#define LGC_PALETTE(pipe) _PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) +#define LGC_PALETTE(pipe, i) _MMIO(_PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) + (i) * 4) #define _GAMMA_MODE_A 0x4a480 #define _GAMMA_MODE_B 0x4ac80 -#define GAMMA_MODE(pipe) _PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B) +#define GAMMA_MODE(pipe) _MMIO_PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B) #define GAMMA_MODE_MODE_MASK (3 << 0) #define GAMMA_MODE_MODE_8BIT (0 << 0) #define GAMMA_MODE_MODE_10BIT (1 << 0) #define GAMMA_MODE_MODE_12BIT (2 << 0) #define GAMMA_MODE_MODE_SPLIT (3 << 0) +/* DMC/CSR */ +#define CSR_PROGRAM(i) _MMIO(0x80000 + (i) * 4) +#define CSR_SSP_BASE_ADDR_GEN9 0x00002FC0 +#define CSR_HTP_ADDR_SKL 0x00500034 +#define CSR_SSP_BASE _MMIO(0x8F074) +#define CSR_HTP_SKL _MMIO(0x8F004) +#define CSR_LAST_WRITE _MMIO(0x8F034) +#define CSR_LAST_WRITE_VALUE 0xc003b400 +/* MMIO address range for CSR program (0x80000 - 0x82FFF) */ +#define CSR_MMIO_START_RANGE 0x80000 +#define CSR_MMIO_END_RANGE 0x8FFFF +#define SKL_CSR_DC3_DC5_COUNT _MMIO(0x80030) +#define SKL_CSR_DC5_DC6_COUNT _MMIO(0x8002C) +#define BXT_CSR_DC3_DC5_COUNT _MMIO(0x80038) + /* interrupts */ #define DE_MASTER_IRQ_CONTROL (1 << 31) #define DE_SPRITEB_FLIP_DONE (1 << 29) @@ -5583,22 +5823,22 @@ enum skl_disp_power_wells { #define DE_PLANEA_FLIP_DONE_IVB (1<<3) #define DE_PLANE_FLIP_DONE_IVB(plane) (1<< (3 + 5*(plane))) #define DE_PIPEA_VBLANK_IVB (1<<0) -#define DE_PIPE_VBLANK_IVB(pipe) (1 << (pipe * 5)) +#define DE_PIPE_VBLANK_IVB(pipe) (1 << ((pipe) * 5)) -#define VLV_MASTER_IER 0x4400c /* Gunit master IER */ +#define VLV_MASTER_IER _MMIO(0x4400c) /* Gunit master IER */ #define MASTER_INTERRUPT_ENABLE (1<<31) -#define DEISR 0x44000 -#define DEIMR 0x44004 -#define DEIIR 0x44008 -#define DEIER 0x4400c +#define DEISR _MMIO(0x44000) +#define DEIMR _MMIO(0x44004) +#define DEIIR _MMIO(0x44008) +#define DEIER _MMIO(0x4400c) -#define GTISR 0x44010 -#define GTIMR 0x44014 -#define GTIIR 0x44018 -#define GTIER 0x4401c +#define GTISR _MMIO(0x44010) +#define GTIMR _MMIO(0x44014) +#define GTIIR _MMIO(0x44018) +#define GTIER _MMIO(0x4401c) -#define GEN8_MASTER_IRQ 0x44200 +#define GEN8_MASTER_IRQ _MMIO(0x44200) #define GEN8_MASTER_IRQ_CONTROL (1<<31) #define GEN8_PCU_IRQ (1<<30) #define GEN8_DE_PCH_IRQ (1<<23) @@ -5607,7 +5847,7 @@ enum skl_disp_power_wells { #define GEN8_DE_PIPE_C_IRQ (1<<18) #define GEN8_DE_PIPE_B_IRQ (1<<17) #define GEN8_DE_PIPE_A_IRQ (1<<16) -#define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+pipe)) +#define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+(pipe))) #define GEN8_GT_VECS_IRQ (1<<6) #define GEN8_GT_PM_IRQ (1<<4) #define GEN8_GT_VCS2_IRQ (1<<3) @@ -5615,21 +5855,22 @@ enum skl_disp_power_wells { #define GEN8_GT_BCS_IRQ (1<<1) #define GEN8_GT_RCS_IRQ (1<<0) -#define GEN8_GT_ISR(which) (0x44300 + (0x10 * (which))) -#define GEN8_GT_IMR(which) (0x44304 + (0x10 * (which))) -#define GEN8_GT_IIR(which) (0x44308 + (0x10 * (which))) -#define GEN8_GT_IER(which) (0x4430c + (0x10 * (which))) +#define GEN8_GT_ISR(which) _MMIO(0x44300 + (0x10 * (which))) +#define GEN8_GT_IMR(which) _MMIO(0x44304 + (0x10 * (which))) +#define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which))) +#define GEN8_GT_IER(which) _MMIO(0x4430c + (0x10 * (which))) -#define GEN8_BCS_IRQ_SHIFT 16 #define GEN8_RCS_IRQ_SHIFT 0 -#define GEN8_VCS2_IRQ_SHIFT 16 +#define GEN8_BCS_IRQ_SHIFT 16 #define GEN8_VCS1_IRQ_SHIFT 0 +#define GEN8_VCS2_IRQ_SHIFT 16 #define GEN8_VECS_IRQ_SHIFT 0 +#define GEN8_WD_IRQ_SHIFT 16 -#define GEN8_DE_PIPE_ISR(pipe) (0x44400 + (0x10 * (pipe))) -#define GEN8_DE_PIPE_IMR(pipe) (0x44404 + (0x10 * (pipe))) -#define GEN8_DE_PIPE_IIR(pipe) (0x44408 + (0x10 * (pipe))) -#define GEN8_DE_PIPE_IER(pipe) (0x4440c + (0x10 * (pipe))) +#define GEN8_DE_PIPE_ISR(pipe) _MMIO(0x44400 + (0x10 * (pipe))) +#define GEN8_DE_PIPE_IMR(pipe) _MMIO(0x44404 + (0x10 * (pipe))) +#define GEN8_DE_PIPE_IIR(pipe) _MMIO(0x44408 + (0x10 * (pipe))) +#define GEN8_DE_PIPE_IER(pipe) _MMIO(0x4440c + (0x10 * (pipe))) #define GEN8_PIPE_FIFO_UNDERRUN (1 << 31) #define GEN8_PIPE_CDCLK_CRC_ERROR (1 << 29) #define GEN8_PIPE_CDCLK_CRC_DONE (1 << 28) @@ -5650,7 +5891,7 @@ enum skl_disp_power_wells { #define GEN9_PIPE_PLANE3_FLIP_DONE (1 << 5) #define GEN9_PIPE_PLANE2_FLIP_DONE (1 << 4) #define GEN9_PIPE_PLANE1_FLIP_DONE (1 << 3) -#define GEN9_PIPE_PLANE_FLIP_DONE(p) (1 << (3 + p)) +#define GEN9_PIPE_PLANE_FLIP_DONE(p) (1 << (3 + (p))) #define GEN8_DE_PIPE_IRQ_FAULT_ERRORS \ (GEN8_PIPE_CURSOR_FAULT | \ GEN8_PIPE_SPRITE_FAULT | \ @@ -5662,10 +5903,10 @@ enum skl_disp_power_wells { GEN9_PIPE_PLANE2_FAULT | \ GEN9_PIPE_PLANE1_FAULT) -#define GEN8_DE_PORT_ISR 0x44440 -#define GEN8_DE_PORT_IMR 0x44444 -#define GEN8_DE_PORT_IIR 0x44448 -#define GEN8_DE_PORT_IER 0x4444c +#define GEN8_DE_PORT_ISR _MMIO(0x44440) +#define GEN8_DE_PORT_IMR _MMIO(0x44444) +#define GEN8_DE_PORT_IIR _MMIO(0x44448) +#define GEN8_DE_PORT_IER _MMIO(0x4444c) #define GEN9_AUX_CHANNEL_D (1 << 27) #define GEN9_AUX_CHANNEL_C (1 << 26) #define GEN9_AUX_CHANNEL_B (1 << 25) @@ -5679,38 +5920,23 @@ enum skl_disp_power_wells { #define BXT_DE_PORT_GMBUS (1 << 1) #define GEN8_AUX_CHANNEL_A (1 << 0) -#define GEN8_DE_MISC_ISR 0x44460 -#define GEN8_DE_MISC_IMR 0x44464 -#define GEN8_DE_MISC_IIR 0x44468 -#define GEN8_DE_MISC_IER 0x4446c +#define GEN8_DE_MISC_ISR _MMIO(0x44460) +#define GEN8_DE_MISC_IMR _MMIO(0x44464) +#define GEN8_DE_MISC_IIR _MMIO(0x44468) +#define GEN8_DE_MISC_IER _MMIO(0x4446c) #define GEN8_DE_MISC_GSE (1 << 27) -#define GEN8_PCU_ISR 0x444e0 -#define GEN8_PCU_IMR 0x444e4 -#define GEN8_PCU_IIR 0x444e8 -#define GEN8_PCU_IER 0x444ec - -/* BXT hotplug control */ -#define BXT_HOTPLUG_CTL 0xC4030 -#define BXT_DDIA_HPD_ENABLE (1 << 28) -#define BXT_DDIA_HPD_STATUS (3 << 24) -#define BXT_DDIC_HPD_ENABLE (1 << 12) -#define BXT_DDIC_HPD_STATUS (3 << 8) -#define BXT_DDIB_HPD_ENABLE (1 << 4) -#define BXT_DDIB_HPD_STATUS (3 << 0) -#define BXT_HOTPLUG_CTL_MASK (BXT_DDIA_HPD_ENABLE | \ - BXT_DDIB_HPD_ENABLE | \ - BXT_DDIC_HPD_ENABLE) -#define BXT_HPD_STATUS_MASK (BXT_DDIA_HPD_STATUS | \ - BXT_DDIB_HPD_STATUS | \ - BXT_DDIC_HPD_STATUS) - -#define ILK_DISPLAY_CHICKEN2 0x42004 +#define GEN8_PCU_ISR _MMIO(0x444e0) +#define GEN8_PCU_IMR _MMIO(0x444e4) +#define GEN8_PCU_IIR _MMIO(0x444e8) +#define GEN8_PCU_IER _MMIO(0x444ec) + +#define ILK_DISPLAY_CHICKEN2 _MMIO(0x42004) /* Required on all Ironlake and Sandybridge according to the B-Spec. */ #define ILK_ELPIN_409_SELECT (1 << 25) #define ILK_DPARB_GATE (1<<22) #define ILK_VSDPFD_FULL (1<<21) -#define FUSE_STRAP 0x42014 +#define FUSE_STRAP _MMIO(0x42014) #define ILK_INTERNAL_GRAPHICS_DISABLE (1 << 31) #define ILK_INTERNAL_DISPLAY_DISABLE (1 << 30) #define ILK_DISPLAY_DEBUG_DISABLE (1 << 29) @@ -5719,18 +5945,18 @@ enum skl_disp_power_wells { #define HSW_CDCLK_LIMIT (1 << 24) #define ILK_DESKTOP (1 << 23) -#define ILK_DSPCLK_GATE_D 0x42020 +#define ILK_DSPCLK_GATE_D _MMIO(0x42020) #define ILK_VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) #define ILK_DPFCUNIT_CLOCK_GATE_DISABLE (1 << 9) #define ILK_DPFCRUNIT_CLOCK_GATE_DISABLE (1 << 8) #define ILK_DPFDUNIT_CLOCK_GATE_ENABLE (1 << 7) #define ILK_DPARBUNIT_CLOCK_GATE_ENABLE (1 << 5) -#define IVB_CHICKEN3 0x4200c +#define IVB_CHICKEN3 _MMIO(0x4200c) # define CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE (1 << 5) # define CHICKEN3_DGMG_DONE_FIX_DISABLE (1 << 2) -#define CHICKEN_PAR1_1 0x42080 +#define CHICKEN_PAR1_1 _MMIO(0x42080) #define DPA_MASK_VBLANK_SRD (1 << 15) #define FORCE_ARB_IDLE_PLANES (1 << 14) @@ -5738,62 +5964,70 @@ enum skl_disp_power_wells { #define _CHICKEN_PIPESL_1_B 0x420b4 #define HSW_FBCQ_DIS (1 << 22) #define BDW_DPRS_MASK_VBLANK_SRD (1 << 0) -#define CHICKEN_PIPESL_1(pipe) _PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B) +#define CHICKEN_PIPESL_1(pipe) _MMIO_PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B) -#define DISP_ARB_CTL 0x45000 +#define DISP_ARB_CTL _MMIO(0x45000) #define DISP_TILE_SURFACE_SWIZZLING (1<<13) #define DISP_FBC_WM_DIS (1<<15) -#define DISP_ARB_CTL2 0x45004 +#define DISP_ARB_CTL2 _MMIO(0x45004) #define DISP_DATA_PARTITION_5_6 (1<<6) -#define DBUF_CTL 0x45008 +#define DBUF_CTL _MMIO(0x45008) #define DBUF_POWER_REQUEST (1<<31) #define DBUF_POWER_STATE (1<<30) -#define GEN7_MSG_CTL 0x45010 +#define GEN7_MSG_CTL _MMIO(0x45010) #define WAIT_FOR_PCH_RESET_ACK (1<<1) #define WAIT_FOR_PCH_FLR_ACK (1<<0) -#define HSW_NDE_RSTWRN_OPT 0x46408 +#define HSW_NDE_RSTWRN_OPT _MMIO(0x46408) #define RESET_PCH_HANDSHAKE_ENABLE (1<<4) -#define FF_SLICE_CS_CHICKEN2 0x20e4 +#define SKL_DFSM _MMIO(0x51000) +#define SKL_DFSM_CDCLK_LIMIT_MASK (3 << 23) +#define SKL_DFSM_CDCLK_LIMIT_675 (0 << 23) +#define SKL_DFSM_CDCLK_LIMIT_540 (1 << 23) +#define SKL_DFSM_CDCLK_LIMIT_450 (2 << 23) +#define SKL_DFSM_CDCLK_LIMIT_337_5 (3 << 23) + +#define FF_SLICE_CS_CHICKEN2 _MMIO(0x20e4) #define GEN9_TSG_BARRIER_ACK_DISABLE (1<<8) /* GEN7 chicken */ -#define GEN7_COMMON_SLICE_CHICKEN1 0x7010 +#define GEN7_COMMON_SLICE_CHICKEN1 _MMIO(0x7010) # define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC ((1<<10) | (1<<26)) # define GEN9_RHWO_OPTIMIZATION_DISABLE (1<<14) -#define COMMON_SLICE_CHICKEN2 0x7014 +#define COMMON_SLICE_CHICKEN2 _MMIO(0x7014) # define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1<<0) -#define HIZ_CHICKEN 0x7018 +#define HIZ_CHICKEN _MMIO(0x7018) # define CHV_HZ_8X8_MODE_IN_1X (1<<15) # define BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE (1<<3) -#define GEN9_SLICE_COMMON_ECO_CHICKEN0 0x7308 +#define GEN9_SLICE_COMMON_ECO_CHICKEN0 _MMIO(0x7308) #define DISABLE_PIXEL_MASK_CAMMING (1<<14) -#define GEN7_L3SQCREG1 0xB010 +#define GEN7_L3SQCREG1 _MMIO(0xB010) #define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 -#define GEN8_L3SQCREG1 0xB100 +#define GEN8_L3SQCREG1 _MMIO(0xB100) #define BDW_WA_L3SQCREG1_DEFAULT 0x784000 -#define GEN7_L3CNTLREG1 0xB01C +#define GEN7_L3CNTLREG1 _MMIO(0xB01C) #define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C47FF8C #define GEN7_L3AGDIS (1<<19) -#define GEN7_L3CNTLREG2 0xB020 -#define GEN7_L3CNTLREG3 0xB024 +#define GEN7_L3CNTLREG2 _MMIO(0xB020) +#define GEN7_L3CNTLREG3 _MMIO(0xB024) -#define GEN7_L3_CHICKEN_MODE_REGISTER 0xB030 +#define GEN7_L3_CHICKEN_MODE_REGISTER _MMIO(0xB030) #define GEN7_WA_L3_CHICKEN_MODE 0x20000000 -#define GEN7_L3SQCREG4 0xb034 +#define GEN7_L3SQCREG4 _MMIO(0xb034) #define L3SQ_URB_READ_CAM_MATCH_DISABLE (1<<27) -#define GEN8_L3SQCREG4 0xb118 +#define GEN8_L3SQCREG4 _MMIO(0xb118) #define GEN8_LQSC_RO_PERF_DIS (1<<27) +#define GEN8_LQSC_FLUSH_COHERENT_LINES (1<<21) /* GEN8 chicken */ -#define HDC_CHICKEN0 0x7300 +#define HDC_CHICKEN0 _MMIO(0x7300) #define HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE (1<<15) #define HDC_FENCE_DEST_SLM_DISABLE (1<<14) #define HDC_DONOT_FETCH_MEM_WHEN_MASKED (1<<11) @@ -5802,17 +6036,17 @@ enum skl_disp_power_wells { #define HDC_BARRIER_PERFORMANCE_DISABLE (1<<10) /* GEN9 chicken */ -#define SLICE_ECO_CHICKEN0 0x7308 +#define SLICE_ECO_CHICKEN0 _MMIO(0x7308) #define PIXEL_MASK_CAMMING_DISABLE (1 << 14) /* WaCatErrorRejectionIssue */ -#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG 0x9030 +#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG _MMIO(0x9030) #define GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB (1<<11) -#define HSW_SCRATCH1 0xb038 +#define HSW_SCRATCH1 _MMIO(0xb038) #define HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE (1<<27) -#define BDW_SCRATCH1 0xb11c +#define BDW_SCRATCH1 _MMIO(0xb11c) #define GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE (1<<2) /* PCH */ @@ -5868,6 +6102,8 @@ enum skl_disp_power_wells { #define SDE_AUXC_CPT (1 << 26) #define SDE_AUXB_CPT (1 << 25) #define SDE_AUX_MASK_CPT (7 << 25) +#define SDE_PORTE_HOTPLUG_SPT (1 << 25) +#define SDE_PORTA_HOTPLUG_SPT (1 << 24) #define SDE_PORTD_HOTPLUG_CPT (1 << 23) #define SDE_PORTC_HOTPLUG_CPT (1 << 22) #define SDE_PORTB_HOTPLUG_CPT (1 << 21) @@ -5878,6 +6114,11 @@ enum skl_disp_power_wells { SDE_PORTD_HOTPLUG_CPT | \ SDE_PORTC_HOTPLUG_CPT | \ SDE_PORTB_HOTPLUG_CPT) +#define SDE_HOTPLUG_MASK_SPT (SDE_PORTE_HOTPLUG_SPT | \ + SDE_PORTD_HOTPLUG_CPT | \ + SDE_PORTC_HOTPLUG_CPT | \ + SDE_PORTB_HOTPLUG_CPT | \ + SDE_PORTA_HOTPLUG_SPT) #define SDE_GMBUS_CPT (1 << 17) #define SDE_ERROR_CPT (1 << 16) #define SDE_AUDIO_CP_REQ_C_CPT (1 << 10) @@ -5899,80 +6140,92 @@ enum skl_disp_power_wells { SDE_FDI_RXB_CPT | \ SDE_FDI_RXA_CPT) -#define SDEISR 0xc4000 -#define SDEIMR 0xc4004 -#define SDEIIR 0xc4008 -#define SDEIER 0xc400c +#define SDEISR _MMIO(0xc4000) +#define SDEIMR _MMIO(0xc4004) +#define SDEIIR _MMIO(0xc4008) +#define SDEIER _MMIO(0xc400c) -#define SERR_INT 0xc4040 +#define SERR_INT _MMIO(0xc4040) #define SERR_INT_POISON (1<<31) #define SERR_INT_TRANS_C_FIFO_UNDERRUN (1<<6) #define SERR_INT_TRANS_B_FIFO_UNDERRUN (1<<3) #define SERR_INT_TRANS_A_FIFO_UNDERRUN (1<<0) -#define SERR_INT_TRANS_FIFO_UNDERRUN(pipe) (1<<(pipe*3)) +#define SERR_INT_TRANS_FIFO_UNDERRUN(pipe) (1<<((pipe)*3)) /* digital port hotplug */ -#define PCH_PORT_HOTPLUG 0xc4030 /* SHOTPLUG_CTL */ -#define PORTD_HOTPLUG_ENABLE (1 << 20) -#define PORTD_PULSE_DURATION_2ms (0) -#define PORTD_PULSE_DURATION_4_5ms (1 << 18) -#define PORTD_PULSE_DURATION_6ms (2 << 18) -#define PORTD_PULSE_DURATION_100ms (3 << 18) -#define PORTD_PULSE_DURATION_MASK (3 << 18) -#define PORTD_HOTPLUG_STATUS_MASK (0x3 << 16) +#define PCH_PORT_HOTPLUG _MMIO(0xc4030) /* SHOTPLUG_CTL */ +#define PORTA_HOTPLUG_ENABLE (1 << 28) /* LPT:LP+ & BXT */ +#define PORTA_HOTPLUG_STATUS_MASK (3 << 24) /* SPT+ & BXT */ +#define PORTA_HOTPLUG_NO_DETECT (0 << 24) /* SPT+ & BXT */ +#define PORTA_HOTPLUG_SHORT_DETECT (1 << 24) /* SPT+ & BXT */ +#define PORTA_HOTPLUG_LONG_DETECT (2 << 24) /* SPT+ & BXT */ +#define PORTD_HOTPLUG_ENABLE (1 << 20) +#define PORTD_PULSE_DURATION_2ms (0 << 18) /* pre-LPT */ +#define PORTD_PULSE_DURATION_4_5ms (1 << 18) /* pre-LPT */ +#define PORTD_PULSE_DURATION_6ms (2 << 18) /* pre-LPT */ +#define PORTD_PULSE_DURATION_100ms (3 << 18) /* pre-LPT */ +#define PORTD_PULSE_DURATION_MASK (3 << 18) /* pre-LPT */ +#define PORTD_HOTPLUG_STATUS_MASK (3 << 16) #define PORTD_HOTPLUG_NO_DETECT (0 << 16) #define PORTD_HOTPLUG_SHORT_DETECT (1 << 16) #define PORTD_HOTPLUG_LONG_DETECT (2 << 16) -#define PORTC_HOTPLUG_ENABLE (1 << 12) -#define PORTC_PULSE_DURATION_2ms (0) -#define PORTC_PULSE_DURATION_4_5ms (1 << 10) -#define PORTC_PULSE_DURATION_6ms (2 << 10) -#define PORTC_PULSE_DURATION_100ms (3 << 10) -#define PORTC_PULSE_DURATION_MASK (3 << 10) -#define PORTC_HOTPLUG_STATUS_MASK (0x3 << 8) +#define PORTC_HOTPLUG_ENABLE (1 << 12) +#define PORTC_PULSE_DURATION_2ms (0 << 10) /* pre-LPT */ +#define PORTC_PULSE_DURATION_4_5ms (1 << 10) /* pre-LPT */ +#define PORTC_PULSE_DURATION_6ms (2 << 10) /* pre-LPT */ +#define PORTC_PULSE_DURATION_100ms (3 << 10) /* pre-LPT */ +#define PORTC_PULSE_DURATION_MASK (3 << 10) /* pre-LPT */ +#define PORTC_HOTPLUG_STATUS_MASK (3 << 8) #define PORTC_HOTPLUG_NO_DETECT (0 << 8) #define PORTC_HOTPLUG_SHORT_DETECT (1 << 8) #define PORTC_HOTPLUG_LONG_DETECT (2 << 8) -#define PORTB_HOTPLUG_ENABLE (1 << 4) -#define PORTB_PULSE_DURATION_2ms (0) -#define PORTB_PULSE_DURATION_4_5ms (1 << 2) -#define PORTB_PULSE_DURATION_6ms (2 << 2) -#define PORTB_PULSE_DURATION_100ms (3 << 2) -#define PORTB_PULSE_DURATION_MASK (3 << 2) -#define PORTB_HOTPLUG_STATUS_MASK (0x3 << 0) +#define PORTB_HOTPLUG_ENABLE (1 << 4) +#define PORTB_PULSE_DURATION_2ms (0 << 2) /* pre-LPT */ +#define PORTB_PULSE_DURATION_4_5ms (1 << 2) /* pre-LPT */ +#define PORTB_PULSE_DURATION_6ms (2 << 2) /* pre-LPT */ +#define PORTB_PULSE_DURATION_100ms (3 << 2) /* pre-LPT */ +#define PORTB_PULSE_DURATION_MASK (3 << 2) /* pre-LPT */ +#define PORTB_HOTPLUG_STATUS_MASK (3 << 0) #define PORTB_HOTPLUG_NO_DETECT (0 << 0) #define PORTB_HOTPLUG_SHORT_DETECT (1 << 0) #define PORTB_HOTPLUG_LONG_DETECT (2 << 0) -#define PCH_GPIOA 0xc5010 -#define PCH_GPIOB 0xc5014 -#define PCH_GPIOC 0xc5018 -#define PCH_GPIOD 0xc501c -#define PCH_GPIOE 0xc5020 -#define PCH_GPIOF 0xc5024 - -#define PCH_GMBUS0 0xc5100 -#define PCH_GMBUS1 0xc5104 -#define PCH_GMBUS2 0xc5108 -#define PCH_GMBUS3 0xc510c -#define PCH_GMBUS4 0xc5110 -#define PCH_GMBUS5 0xc5120 +#define PCH_PORT_HOTPLUG2 _MMIO(0xc403C) /* SHOTPLUG_CTL2 SPT+ */ +#define PORTE_HOTPLUG_ENABLE (1 << 4) +#define PORTE_HOTPLUG_STATUS_MASK (3 << 0) +#define PORTE_HOTPLUG_NO_DETECT (0 << 0) +#define PORTE_HOTPLUG_SHORT_DETECT (1 << 0) +#define PORTE_HOTPLUG_LONG_DETECT (2 << 0) + +#define PCH_GPIOA _MMIO(0xc5010) +#define PCH_GPIOB _MMIO(0xc5014) +#define PCH_GPIOC _MMIO(0xc5018) +#define PCH_GPIOD _MMIO(0xc501c) +#define PCH_GPIOE _MMIO(0xc5020) +#define PCH_GPIOF _MMIO(0xc5024) + +#define PCH_GMBUS0 _MMIO(0xc5100) +#define PCH_GMBUS1 _MMIO(0xc5104) +#define PCH_GMBUS2 _MMIO(0xc5108) +#define PCH_GMBUS3 _MMIO(0xc510c) +#define PCH_GMBUS4 _MMIO(0xc5110) +#define PCH_GMBUS5 _MMIO(0xc5120) #define _PCH_DPLL_A 0xc6014 #define _PCH_DPLL_B 0xc6018 -#define PCH_DPLL(pll) (pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B) +#define PCH_DPLL(pll) _MMIO(pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B) #define _PCH_FPA0 0xc6040 #define FP_CB_TUNE (0x3<<22) #define _PCH_FPA1 0xc6044 #define _PCH_FPB0 0xc6048 #define _PCH_FPB1 0xc604c -#define PCH_FP0(pll) (pll == 0 ? _PCH_FPA0 : _PCH_FPB0) -#define PCH_FP1(pll) (pll == 0 ? _PCH_FPA1 : _PCH_FPB1) +#define PCH_FP0(pll) _MMIO(pll == 0 ? _PCH_FPA0 : _PCH_FPB0) +#define PCH_FP1(pll) _MMIO(pll == 0 ? _PCH_FPA1 : _PCH_FPB1) -#define PCH_DPLL_TEST 0xc606c +#define PCH_DPLL_TEST _MMIO(0xc606c) -#define PCH_DREF_CONTROL 0xC6200 +#define PCH_DREF_CONTROL _MMIO(0xC6200) #define DREF_CONTROL_MASK 0x7fc3 #define DREF_CPU_SOURCE_OUTPUT_DISABLE (0<<13) #define DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD (2<<13) @@ -5995,22 +6248,22 @@ enum skl_disp_power_wells { #define DREF_SSC4_DISABLE (0) #define DREF_SSC4_ENABLE (1) -#define PCH_RAWCLK_FREQ 0xc6204 +#define PCH_RAWCLK_FREQ _MMIO(0xc6204) #define FDL_TP1_TIMER_SHIFT 12 #define FDL_TP1_TIMER_MASK (3<<12) #define FDL_TP2_TIMER_SHIFT 10 #define FDL_TP2_TIMER_MASK (3<<10) #define RAWCLK_FREQ_MASK 0x3ff -#define PCH_DPLL_TMR_CFG 0xc6208 +#define PCH_DPLL_TMR_CFG _MMIO(0xc6208) -#define PCH_SSC4_PARMS 0xc6210 -#define PCH_SSC4_AUX_PARMS 0xc6214 +#define PCH_SSC4_PARMS _MMIO(0xc6210) +#define PCH_SSC4_AUX_PARMS _MMIO(0xc6214) -#define PCH_DPLL_SEL 0xc7000 -#define TRANS_DPLLB_SEL(pipe) (1 << (pipe * 4)) +#define PCH_DPLL_SEL _MMIO(0xc7000) +#define TRANS_DPLLB_SEL(pipe) (1 << ((pipe) * 4)) #define TRANS_DPLLA_SEL(pipe) 0 -#define TRANS_DPLL_ENABLE(pipe) (1 << (pipe * 4 + 3)) +#define TRANS_DPLL_ENABLE(pipe) (1 << ((pipe) * 4 + 3)) /* transcoder */ @@ -6047,84 +6300,81 @@ enum skl_disp_power_wells { #define _VIDEO_DIP_CTL_A 0xe0200 #define _VIDEO_DIP_DATA_A 0xe0208 #define _VIDEO_DIP_GCP_A 0xe0210 +#define GCP_COLOR_INDICATION (1 << 2) +#define GCP_DEFAULT_PHASE_ENABLE (1 << 1) +#define GCP_AV_MUTE (1 << 0) #define _VIDEO_DIP_CTL_B 0xe1200 #define _VIDEO_DIP_DATA_B 0xe1208 #define _VIDEO_DIP_GCP_B 0xe1210 -#define TVIDEO_DIP_CTL(pipe) _PIPE(pipe, _VIDEO_DIP_CTL_A, _VIDEO_DIP_CTL_B) -#define TVIDEO_DIP_DATA(pipe) _PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B) -#define TVIDEO_DIP_GCP(pipe) _PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B) +#define TVIDEO_DIP_CTL(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_CTL_A, _VIDEO_DIP_CTL_B) +#define TVIDEO_DIP_DATA(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B) +#define TVIDEO_DIP_GCP(pipe) _MMIO_PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B) /* Per-transcoder DIP controls (VLV) */ -#define VLV_VIDEO_DIP_CTL_A (VLV_DISPLAY_BASE + 0x60200) -#define VLV_VIDEO_DIP_DATA_A (VLV_DISPLAY_BASE + 0x60208) -#define VLV_VIDEO_DIP_GDCP_PAYLOAD_A (VLV_DISPLAY_BASE + 0x60210) +#define _VLV_VIDEO_DIP_CTL_A (VLV_DISPLAY_BASE + 0x60200) +#define _VLV_VIDEO_DIP_DATA_A (VLV_DISPLAY_BASE + 0x60208) +#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_A (VLV_DISPLAY_BASE + 0x60210) -#define VLV_VIDEO_DIP_CTL_B (VLV_DISPLAY_BASE + 0x61170) -#define VLV_VIDEO_DIP_DATA_B (VLV_DISPLAY_BASE + 0x61174) -#define VLV_VIDEO_DIP_GDCP_PAYLOAD_B (VLV_DISPLAY_BASE + 0x61178) +#define _VLV_VIDEO_DIP_CTL_B (VLV_DISPLAY_BASE + 0x61170) +#define _VLV_VIDEO_DIP_DATA_B (VLV_DISPLAY_BASE + 0x61174) +#define _VLV_VIDEO_DIP_GDCP_PAYLOAD_B (VLV_DISPLAY_BASE + 0x61178) -#define CHV_VIDEO_DIP_CTL_C (VLV_DISPLAY_BASE + 0x611f0) -#define CHV_VIDEO_DIP_DATA_C (VLV_DISPLAY_BASE + 0x611f4) -#define CHV_VIDEO_DIP_GDCP_PAYLOAD_C (VLV_DISPLAY_BASE + 0x611f8) +#define _CHV_VIDEO_DIP_CTL_C (VLV_DISPLAY_BASE + 0x611f0) +#define _CHV_VIDEO_DIP_DATA_C (VLV_DISPLAY_BASE + 0x611f4) +#define _CHV_VIDEO_DIP_GDCP_PAYLOAD_C (VLV_DISPLAY_BASE + 0x611f8) #define VLV_TVIDEO_DIP_CTL(pipe) \ - _PIPE3((pipe), VLV_VIDEO_DIP_CTL_A, \ - VLV_VIDEO_DIP_CTL_B, CHV_VIDEO_DIP_CTL_C) + _MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_CTL_A, \ + _VLV_VIDEO_DIP_CTL_B, _CHV_VIDEO_DIP_CTL_C) #define VLV_TVIDEO_DIP_DATA(pipe) \ - _PIPE3((pipe), VLV_VIDEO_DIP_DATA_A, \ - VLV_VIDEO_DIP_DATA_B, CHV_VIDEO_DIP_DATA_C) + _MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_DATA_A, \ + _VLV_VIDEO_DIP_DATA_B, _CHV_VIDEO_DIP_DATA_C) #define VLV_TVIDEO_DIP_GCP(pipe) \ - _PIPE3((pipe), VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \ - VLV_VIDEO_DIP_GDCP_PAYLOAD_B, CHV_VIDEO_DIP_GDCP_PAYLOAD_C) + _MMIO_PIPE3((pipe), _VLV_VIDEO_DIP_GDCP_PAYLOAD_A, \ + _VLV_VIDEO_DIP_GDCP_PAYLOAD_B, _CHV_VIDEO_DIP_GDCP_PAYLOAD_C) /* Haswell DIP controls */ -#define HSW_VIDEO_DIP_CTL_A 0x60200 -#define HSW_VIDEO_DIP_AVI_DATA_A 0x60220 -#define HSW_VIDEO_DIP_VS_DATA_A 0x60260 -#define HSW_VIDEO_DIP_SPD_DATA_A 0x602A0 -#define HSW_VIDEO_DIP_GMP_DATA_A 0x602E0 -#define HSW_VIDEO_DIP_VSC_DATA_A 0x60320 -#define HSW_VIDEO_DIP_AVI_ECC_A 0x60240 -#define HSW_VIDEO_DIP_VS_ECC_A 0x60280 -#define HSW_VIDEO_DIP_SPD_ECC_A 0x602C0 -#define HSW_VIDEO_DIP_GMP_ECC_A 0x60300 -#define HSW_VIDEO_DIP_VSC_ECC_A 0x60344 -#define HSW_VIDEO_DIP_GCP_A 0x60210 - -#define HSW_VIDEO_DIP_CTL_B 0x61200 -#define HSW_VIDEO_DIP_AVI_DATA_B 0x61220 -#define HSW_VIDEO_DIP_VS_DATA_B 0x61260 -#define HSW_VIDEO_DIP_SPD_DATA_B 0x612A0 -#define HSW_VIDEO_DIP_GMP_DATA_B 0x612E0 -#define HSW_VIDEO_DIP_VSC_DATA_B 0x61320 -#define HSW_VIDEO_DIP_BVI_ECC_B 0x61240 -#define HSW_VIDEO_DIP_VS_ECC_B 0x61280 -#define HSW_VIDEO_DIP_SPD_ECC_B 0x612C0 -#define HSW_VIDEO_DIP_GMP_ECC_B 0x61300 -#define HSW_VIDEO_DIP_VSC_ECC_B 0x61344 -#define HSW_VIDEO_DIP_GCP_B 0x61210 - -#define HSW_TVIDEO_DIP_CTL(trans) \ - _TRANSCODER2(trans, HSW_VIDEO_DIP_CTL_A) -#define HSW_TVIDEO_DIP_AVI_DATA(trans) \ - _TRANSCODER2(trans, HSW_VIDEO_DIP_AVI_DATA_A) -#define HSW_TVIDEO_DIP_VS_DATA(trans) \ - _TRANSCODER2(trans, HSW_VIDEO_DIP_VS_DATA_A) -#define HSW_TVIDEO_DIP_SPD_DATA(trans) \ - _TRANSCODER2(trans, HSW_VIDEO_DIP_SPD_DATA_A) -#define HSW_TVIDEO_DIP_GCP(trans) \ - _TRANSCODER2(trans, HSW_VIDEO_DIP_GCP_A) -#define HSW_TVIDEO_DIP_VSC_DATA(trans) \ - _TRANSCODER2(trans, HSW_VIDEO_DIP_VSC_DATA_A) - -#define HSW_STEREO_3D_CTL_A 0x70020 -#define S3D_ENABLE (1<<31) -#define HSW_STEREO_3D_CTL_B 0x71020 - -#define HSW_STEREO_3D_CTL(trans) \ - _PIPE2(trans, HSW_STEREO_3D_CTL_A) + +#define _HSW_VIDEO_DIP_CTL_A 0x60200 +#define _HSW_VIDEO_DIP_AVI_DATA_A 0x60220 +#define _HSW_VIDEO_DIP_VS_DATA_A 0x60260 +#define _HSW_VIDEO_DIP_SPD_DATA_A 0x602A0 +#define _HSW_VIDEO_DIP_GMP_DATA_A 0x602E0 +#define _HSW_VIDEO_DIP_VSC_DATA_A 0x60320 +#define _HSW_VIDEO_DIP_AVI_ECC_A 0x60240 +#define _HSW_VIDEO_DIP_VS_ECC_A 0x60280 +#define _HSW_VIDEO_DIP_SPD_ECC_A 0x602C0 +#define _HSW_VIDEO_DIP_GMP_ECC_A 0x60300 +#define _HSW_VIDEO_DIP_VSC_ECC_A 0x60344 +#define _HSW_VIDEO_DIP_GCP_A 0x60210 + +#define _HSW_VIDEO_DIP_CTL_B 0x61200 +#define _HSW_VIDEO_DIP_AVI_DATA_B 0x61220 +#define _HSW_VIDEO_DIP_VS_DATA_B 0x61260 +#define _HSW_VIDEO_DIP_SPD_DATA_B 0x612A0 +#define _HSW_VIDEO_DIP_GMP_DATA_B 0x612E0 +#define _HSW_VIDEO_DIP_VSC_DATA_B 0x61320 +#define _HSW_VIDEO_DIP_BVI_ECC_B 0x61240 +#define _HSW_VIDEO_DIP_VS_ECC_B 0x61280 +#define _HSW_VIDEO_DIP_SPD_ECC_B 0x612C0 +#define _HSW_VIDEO_DIP_GMP_ECC_B 0x61300 +#define _HSW_VIDEO_DIP_VSC_ECC_B 0x61344 +#define _HSW_VIDEO_DIP_GCP_B 0x61210 + +#define HSW_TVIDEO_DIP_CTL(trans) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_CTL_A) +#define HSW_TVIDEO_DIP_AVI_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4) +#define HSW_TVIDEO_DIP_VS_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4) +#define HSW_TVIDEO_DIP_SPD_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4) +#define HSW_TVIDEO_DIP_GCP(trans) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A) +#define HSW_TVIDEO_DIP_VSC_DATA(trans, i) _MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4) + +#define _HSW_STEREO_3D_CTL_A 0x70020 +#define S3D_ENABLE (1<<31) +#define _HSW_STEREO_3D_CTL_B 0x71020 + +#define HSW_STEREO_3D_CTL(trans) _MMIO_PIPE2(trans, _HSW_STEREO_3D_CTL_A) #define _PCH_TRANS_HTOTAL_B 0xe1000 #define _PCH_TRANS_HBLANK_B 0xe1004 @@ -6132,16 +6382,15 @@ enum skl_disp_power_wells { #define _PCH_TRANS_VTOTAL_B 0xe100c #define _PCH_TRANS_VBLANK_B 0xe1010 #define _PCH_TRANS_VSYNC_B 0xe1014 -#define _PCH_TRANS_VSYNCSHIFT_B 0xe1028 +#define _PCH_TRANS_VSYNCSHIFT_B 0xe1028 -#define PCH_TRANS_HTOTAL(pipe) _PIPE(pipe, _PCH_TRANS_HTOTAL_A, _PCH_TRANS_HTOTAL_B) -#define PCH_TRANS_HBLANK(pipe) _PIPE(pipe, _PCH_TRANS_HBLANK_A, _PCH_TRANS_HBLANK_B) -#define PCH_TRANS_HSYNC(pipe) _PIPE(pipe, _PCH_TRANS_HSYNC_A, _PCH_TRANS_HSYNC_B) -#define PCH_TRANS_VTOTAL(pipe) _PIPE(pipe, _PCH_TRANS_VTOTAL_A, _PCH_TRANS_VTOTAL_B) -#define PCH_TRANS_VBLANK(pipe) _PIPE(pipe, _PCH_TRANS_VBLANK_A, _PCH_TRANS_VBLANK_B) -#define PCH_TRANS_VSYNC(pipe) _PIPE(pipe, _PCH_TRANS_VSYNC_A, _PCH_TRANS_VSYNC_B) -#define PCH_TRANS_VSYNCSHIFT(pipe) _PIPE(pipe, _PCH_TRANS_VSYNCSHIFT_A, \ - _PCH_TRANS_VSYNCSHIFT_B) +#define PCH_TRANS_HTOTAL(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_HTOTAL_A, _PCH_TRANS_HTOTAL_B) +#define PCH_TRANS_HBLANK(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_HBLANK_A, _PCH_TRANS_HBLANK_B) +#define PCH_TRANS_HSYNC(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_HSYNC_A, _PCH_TRANS_HSYNC_B) +#define PCH_TRANS_VTOTAL(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VTOTAL_A, _PCH_TRANS_VTOTAL_B) +#define PCH_TRANS_VBLANK(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VBLANK_A, _PCH_TRANS_VBLANK_B) +#define PCH_TRANS_VSYNC(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VSYNC_A, _PCH_TRANS_VSYNC_B) +#define PCH_TRANS_VSYNCSHIFT(pipe) _MMIO_PIPE(pipe, _PCH_TRANS_VSYNCSHIFT_A, _PCH_TRANS_VSYNCSHIFT_B) #define _PCH_TRANSB_DATA_M1 0xe1030 #define _PCH_TRANSB_DATA_N1 0xe1034 @@ -6152,19 +6401,19 @@ enum skl_disp_power_wells { #define _PCH_TRANSB_LINK_M2 0xe1048 #define _PCH_TRANSB_LINK_N2 0xe104c -#define PCH_TRANS_DATA_M1(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_M1, _PCH_TRANSB_DATA_M1) -#define PCH_TRANS_DATA_N1(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_N1, _PCH_TRANSB_DATA_N1) -#define PCH_TRANS_DATA_M2(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_M2, _PCH_TRANSB_DATA_M2) -#define PCH_TRANS_DATA_N2(pipe) _PIPE(pipe, _PCH_TRANSA_DATA_N2, _PCH_TRANSB_DATA_N2) -#define PCH_TRANS_LINK_M1(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_M1, _PCH_TRANSB_LINK_M1) -#define PCH_TRANS_LINK_N1(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_N1, _PCH_TRANSB_LINK_N1) -#define PCH_TRANS_LINK_M2(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_M2, _PCH_TRANSB_LINK_M2) -#define PCH_TRANS_LINK_N2(pipe) _PIPE(pipe, _PCH_TRANSA_LINK_N2, _PCH_TRANSB_LINK_N2) +#define PCH_TRANS_DATA_M1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_M1, _PCH_TRANSB_DATA_M1) +#define PCH_TRANS_DATA_N1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_N1, _PCH_TRANSB_DATA_N1) +#define PCH_TRANS_DATA_M2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_M2, _PCH_TRANSB_DATA_M2) +#define PCH_TRANS_DATA_N2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_DATA_N2, _PCH_TRANSB_DATA_N2) +#define PCH_TRANS_LINK_M1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_M1, _PCH_TRANSB_LINK_M1) +#define PCH_TRANS_LINK_N1(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_N1, _PCH_TRANSB_LINK_N1) +#define PCH_TRANS_LINK_M2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_M2, _PCH_TRANSB_LINK_M2) +#define PCH_TRANS_LINK_N2(pipe) _MMIO_PIPE(pipe, _PCH_TRANSA_LINK_N2, _PCH_TRANSB_LINK_N2) #define _PCH_TRANSACONF 0xf0008 #define _PCH_TRANSBCONF 0xf1008 -#define PCH_TRANSCONF(pipe) _PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF) -#define LPT_TRANSCONF _PCH_TRANSACONF /* lpt has only one transcoder */ +#define PCH_TRANSCONF(pipe) _MMIO_PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF) +#define LPT_TRANSCONF PCH_TRANSCONF(PIPE_A) /* lpt has only one transcoder */ #define TRANS_DISABLE (0<<31) #define TRANS_ENABLE (1<<31) #define TRANS_STATE_MASK (1<<30) @@ -6185,44 +6434,47 @@ enum skl_disp_power_wells { #define _TRANSA_CHICKEN1 0xf0060 #define _TRANSB_CHICKEN1 0xf1060 -#define TRANS_CHICKEN1(pipe) _PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1) +#define TRANS_CHICKEN1(pipe) _MMIO_PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1) +#define TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE (1<<10) #define TRANS_CHICKEN1_DP0UNIT_GC_DISABLE (1<<4) #define _TRANSA_CHICKEN2 0xf0064 #define _TRANSB_CHICKEN2 0xf1064 -#define TRANS_CHICKEN2(pipe) _PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2) +#define TRANS_CHICKEN2(pipe) _MMIO_PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2) #define TRANS_CHICKEN2_TIMING_OVERRIDE (1<<31) #define TRANS_CHICKEN2_FDI_POLARITY_REVERSED (1<<29) #define TRANS_CHICKEN2_FRAME_START_DELAY_MASK (3<<27) #define TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER (1<<26) #define TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH (1<<25) -#define SOUTH_CHICKEN1 0xc2000 +#define SOUTH_CHICKEN1 _MMIO(0xc2000) #define FDIA_PHASE_SYNC_SHIFT_OVR 19 #define FDIA_PHASE_SYNC_SHIFT_EN 18 #define FDI_PHASE_SYNC_OVR(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2))) #define FDI_PHASE_SYNC_EN(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2))) #define FDI_BC_BIFURCATION_SELECT (1 << 12) -#define SOUTH_CHICKEN2 0xc2004 +#define SPT_PWM_GRANULARITY (1<<0) +#define SOUTH_CHICKEN2 _MMIO(0xc2004) #define FDI_MPHY_IOSFSB_RESET_STATUS (1<<13) #define FDI_MPHY_IOSFSB_RESET_CTL (1<<12) +#define LPT_PWM_GRANULARITY (1<<5) #define DPLS_EDP_PPS_FIX_DIS (1<<0) -#define _FDI_RXA_CHICKEN 0xc200c -#define _FDI_RXB_CHICKEN 0xc2010 +#define _FDI_RXA_CHICKEN 0xc200c +#define _FDI_RXB_CHICKEN 0xc2010 #define FDI_RX_PHASE_SYNC_POINTER_OVR (1<<1) #define FDI_RX_PHASE_SYNC_POINTER_EN (1<<0) -#define FDI_RX_CHICKEN(pipe) _PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN) +#define FDI_RX_CHICKEN(pipe) _MMIO_PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN) -#define SOUTH_DSPCLK_GATE_D 0xc2020 +#define SOUTH_DSPCLK_GATE_D _MMIO(0xc2020) #define PCH_DPLUNIT_CLOCK_GATE_DISABLE (1<<30) #define PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1<<29) #define PCH_CPUNIT_CLOCK_GATE_DISABLE (1<<14) #define PCH_LP_PARTITION_LEVEL_DISABLE (1<<12) /* CPU: FDI_TX */ -#define _FDI_TXA_CTL 0x60100 -#define _FDI_TXB_CTL 0x61100 -#define FDI_TX_CTL(pipe) _PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL) +#define _FDI_TXA_CTL 0x60100 +#define _FDI_TXB_CTL 0x61100 +#define FDI_TX_CTL(pipe) _MMIO_PIPE(pipe, _FDI_TXA_CTL, _FDI_TXB_CTL) #define FDI_TX_DISABLE (0<<31) #define FDI_TX_ENABLE (1<<31) #define FDI_LINK_TRAIN_PATTERN_1 (0<<28) @@ -6272,7 +6524,7 @@ enum skl_disp_power_wells { /* FDI_RX, FDI_X is hard-wired to Transcoder_X */ #define _FDI_RXA_CTL 0xf000c #define _FDI_RXB_CTL 0xf100c -#define FDI_RX_CTL(pipe) _PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL) +#define FDI_RX_CTL(pipe) _MMIO_PIPE(pipe, _FDI_RXA_CTL, _FDI_RXB_CTL) #define FDI_RX_ENABLE (1<<31) /* train, dp width same as FDI_TX */ #define FDI_FS_ERRC_ENABLE (1<<27) @@ -6308,14 +6560,14 @@ enum skl_disp_power_wells { #define FDI_RX_TP1_TO_TP2_48 (2<<20) #define FDI_RX_TP1_TO_TP2_64 (3<<20) #define FDI_RX_FDI_DELAY_90 (0x90<<0) -#define FDI_RX_MISC(pipe) _PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC) +#define FDI_RX_MISC(pipe) _MMIO_PIPE(pipe, _FDI_RXA_MISC, _FDI_RXB_MISC) -#define _FDI_RXA_TUSIZE1 0xf0030 -#define _FDI_RXA_TUSIZE2 0xf0038 -#define _FDI_RXB_TUSIZE1 0xf1030 -#define _FDI_RXB_TUSIZE2 0xf1038 -#define FDI_RX_TUSIZE1(pipe) _PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1) -#define FDI_RX_TUSIZE2(pipe) _PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2) +#define _FDI_RXA_TUSIZE1 0xf0030 +#define _FDI_RXA_TUSIZE2 0xf0038 +#define _FDI_RXB_TUSIZE1 0xf1030 +#define _FDI_RXB_TUSIZE2 0xf1038 +#define FDI_RX_TUSIZE1(pipe) _MMIO_PIPE(pipe, _FDI_RXA_TUSIZE1, _FDI_RXB_TUSIZE1) +#define FDI_RX_TUSIZE2(pipe) _MMIO_PIPE(pipe, _FDI_RXA_TUSIZE2, _FDI_RXB_TUSIZE2) /* FDI_RX interrupt register format */ #define FDI_RX_INTER_LANE_ALIGN (1<<10) @@ -6330,52 +6582,51 @@ enum skl_disp_power_wells { #define FDI_RX_CROSS_CLOCK_OVERFLOW (1<<1) #define FDI_RX_SYMBOL_QUEUE_OVERFLOW (1<<0) -#define _FDI_RXA_IIR 0xf0014 -#define _FDI_RXA_IMR 0xf0018 -#define _FDI_RXB_IIR 0xf1014 -#define _FDI_RXB_IMR 0xf1018 -#define FDI_RX_IIR(pipe) _PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR) -#define FDI_RX_IMR(pipe) _PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR) +#define _FDI_RXA_IIR 0xf0014 +#define _FDI_RXA_IMR 0xf0018 +#define _FDI_RXB_IIR 0xf1014 +#define _FDI_RXB_IMR 0xf1018 +#define FDI_RX_IIR(pipe) _MMIO_PIPE(pipe, _FDI_RXA_IIR, _FDI_RXB_IIR) +#define FDI_RX_IMR(pipe) _MMIO_PIPE(pipe, _FDI_RXA_IMR, _FDI_RXB_IMR) -#define FDI_PLL_CTL_1 0xfe000 -#define FDI_PLL_CTL_2 0xfe004 +#define FDI_PLL_CTL_1 _MMIO(0xfe000) +#define FDI_PLL_CTL_2 _MMIO(0xfe004) -#define PCH_LVDS 0xe1180 +#define PCH_LVDS _MMIO(0xe1180) #define LVDS_DETECTED (1 << 1) /* vlv has 2 sets of panel control regs. */ -#define PIPEA_PP_STATUS (VLV_DISPLAY_BASE + 0x61200) -#define PIPEA_PP_CONTROL (VLV_DISPLAY_BASE + 0x61204) -#define PIPEA_PP_ON_DELAYS (VLV_DISPLAY_BASE + 0x61208) +#define _PIPEA_PP_STATUS (VLV_DISPLAY_BASE + 0x61200) +#define _PIPEA_PP_CONTROL (VLV_DISPLAY_BASE + 0x61204) +#define _PIPEA_PP_ON_DELAYS (VLV_DISPLAY_BASE + 0x61208) #define PANEL_PORT_SELECT_VLV(port) ((port) << 30) -#define PIPEA_PP_OFF_DELAYS (VLV_DISPLAY_BASE + 0x6120c) -#define PIPEA_PP_DIVISOR (VLV_DISPLAY_BASE + 0x61210) - -#define PIPEB_PP_STATUS (VLV_DISPLAY_BASE + 0x61300) -#define PIPEB_PP_CONTROL (VLV_DISPLAY_BASE + 0x61304) -#define PIPEB_PP_ON_DELAYS (VLV_DISPLAY_BASE + 0x61308) -#define PIPEB_PP_OFF_DELAYS (VLV_DISPLAY_BASE + 0x6130c) -#define PIPEB_PP_DIVISOR (VLV_DISPLAY_BASE + 0x61310) - -#define VLV_PIPE_PP_STATUS(pipe) _PIPE(pipe, PIPEA_PP_STATUS, PIPEB_PP_STATUS) -#define VLV_PIPE_PP_CONTROL(pipe) _PIPE(pipe, PIPEA_PP_CONTROL, PIPEB_PP_CONTROL) -#define VLV_PIPE_PP_ON_DELAYS(pipe) \ - _PIPE(pipe, PIPEA_PP_ON_DELAYS, PIPEB_PP_ON_DELAYS) -#define VLV_PIPE_PP_OFF_DELAYS(pipe) \ - _PIPE(pipe, PIPEA_PP_OFF_DELAYS, PIPEB_PP_OFF_DELAYS) -#define VLV_PIPE_PP_DIVISOR(pipe) \ - _PIPE(pipe, PIPEA_PP_DIVISOR, PIPEB_PP_DIVISOR) - -#define PCH_PP_STATUS 0xc7200 -#define PCH_PP_CONTROL 0xc7204 +#define _PIPEA_PP_OFF_DELAYS (VLV_DISPLAY_BASE + 0x6120c) +#define _PIPEA_PP_DIVISOR (VLV_DISPLAY_BASE + 0x61210) + +#define _PIPEB_PP_STATUS (VLV_DISPLAY_BASE + 0x61300) +#define _PIPEB_PP_CONTROL (VLV_DISPLAY_BASE + 0x61304) +#define _PIPEB_PP_ON_DELAYS (VLV_DISPLAY_BASE + 0x61308) +#define _PIPEB_PP_OFF_DELAYS (VLV_DISPLAY_BASE + 0x6130c) +#define _PIPEB_PP_DIVISOR (VLV_DISPLAY_BASE + 0x61310) + +#define VLV_PIPE_PP_STATUS(pipe) _MMIO_PIPE(pipe, _PIPEA_PP_STATUS, _PIPEB_PP_STATUS) +#define VLV_PIPE_PP_CONTROL(pipe) _MMIO_PIPE(pipe, _PIPEA_PP_CONTROL, _PIPEB_PP_CONTROL) +#define VLV_PIPE_PP_ON_DELAYS(pipe) _MMIO_PIPE(pipe, _PIPEA_PP_ON_DELAYS, _PIPEB_PP_ON_DELAYS) +#define VLV_PIPE_PP_OFF_DELAYS(pipe) _MMIO_PIPE(pipe, _PIPEA_PP_OFF_DELAYS, _PIPEB_PP_OFF_DELAYS) +#define VLV_PIPE_PP_DIVISOR(pipe) _MMIO_PIPE(pipe, _PIPEA_PP_DIVISOR, _PIPEB_PP_DIVISOR) + +#define _PCH_PP_STATUS 0xc7200 +#define _PCH_PP_CONTROL 0xc7204 #define PANEL_UNLOCK_REGS (0xabcd << 16) #define PANEL_UNLOCK_MASK (0xffff << 16) +#define BXT_POWER_CYCLE_DELAY_MASK (0x1f0) +#define BXT_POWER_CYCLE_DELAY_SHIFT 4 #define EDP_FORCE_VDD (1 << 3) #define EDP_BLC_ENABLE (1 << 2) #define PANEL_POWER_RESET (1 << 1) #define PANEL_POWER_OFF (0 << 0) #define PANEL_POWER_ON (1 << 0) -#define PCH_PP_ON_DELAYS 0xc7208 +#define _PCH_PP_ON_DELAYS 0xc7208 #define PANEL_PORT_SELECT_MASK (3 << 30) #define PANEL_PORT_SELECT_LVDS (0 << 30) #define PANEL_PORT_SELECT_DPA (1 << 30) @@ -6386,41 +6637,64 @@ enum skl_disp_power_wells { #define PANEL_LIGHT_ON_DELAY_MASK (0x1fff) #define PANEL_LIGHT_ON_DELAY_SHIFT 0 -#define PCH_PP_OFF_DELAYS 0xc720c +#define _PCH_PP_OFF_DELAYS 0xc720c #define PANEL_POWER_DOWN_DELAY_MASK (0x1fff0000) #define PANEL_POWER_DOWN_DELAY_SHIFT 16 #define PANEL_LIGHT_OFF_DELAY_MASK (0x1fff) #define PANEL_LIGHT_OFF_DELAY_SHIFT 0 -#define PCH_PP_DIVISOR 0xc7210 +#define _PCH_PP_DIVISOR 0xc7210 #define PP_REFERENCE_DIVIDER_MASK (0xffffff00) #define PP_REFERENCE_DIVIDER_SHIFT 8 #define PANEL_POWER_CYCLE_DELAY_MASK (0x1f) #define PANEL_POWER_CYCLE_DELAY_SHIFT 0 -#define PCH_DP_B 0xe4100 -#define PCH_DPB_AUX_CH_CTL 0xe4110 -#define PCH_DPB_AUX_CH_DATA1 0xe4114 -#define PCH_DPB_AUX_CH_DATA2 0xe4118 -#define PCH_DPB_AUX_CH_DATA3 0xe411c -#define PCH_DPB_AUX_CH_DATA4 0xe4120 -#define PCH_DPB_AUX_CH_DATA5 0xe4124 - -#define PCH_DP_C 0xe4200 -#define PCH_DPC_AUX_CH_CTL 0xe4210 -#define PCH_DPC_AUX_CH_DATA1 0xe4214 -#define PCH_DPC_AUX_CH_DATA2 0xe4218 -#define PCH_DPC_AUX_CH_DATA3 0xe421c -#define PCH_DPC_AUX_CH_DATA4 0xe4220 -#define PCH_DPC_AUX_CH_DATA5 0xe4224 - -#define PCH_DP_D 0xe4300 -#define PCH_DPD_AUX_CH_CTL 0xe4310 -#define PCH_DPD_AUX_CH_DATA1 0xe4314 -#define PCH_DPD_AUX_CH_DATA2 0xe4318 -#define PCH_DPD_AUX_CH_DATA3 0xe431c -#define PCH_DPD_AUX_CH_DATA4 0xe4320 -#define PCH_DPD_AUX_CH_DATA5 0xe4324 +#define PCH_PP_STATUS _MMIO(_PCH_PP_STATUS) +#define PCH_PP_CONTROL _MMIO(_PCH_PP_CONTROL) +#define PCH_PP_ON_DELAYS _MMIO(_PCH_PP_ON_DELAYS) +#define PCH_PP_OFF_DELAYS _MMIO(_PCH_PP_OFF_DELAYS) +#define PCH_PP_DIVISOR _MMIO(_PCH_PP_DIVISOR) + +/* BXT PPS changes - 2nd set of PPS registers */ +#define _BXT_PP_STATUS2 0xc7300 +#define _BXT_PP_CONTROL2 0xc7304 +#define _BXT_PP_ON_DELAYS2 0xc7308 +#define _BXT_PP_OFF_DELAYS2 0xc730c + +#define BXT_PP_STATUS(n) _MMIO_PIPE(n, _PCH_PP_STATUS, _BXT_PP_STATUS2) +#define BXT_PP_CONTROL(n) _MMIO_PIPE(n, _PCH_PP_CONTROL, _BXT_PP_CONTROL2) +#define BXT_PP_ON_DELAYS(n) _MMIO_PIPE(n, _PCH_PP_ON_DELAYS, _BXT_PP_ON_DELAYS2) +#define BXT_PP_OFF_DELAYS(n) _MMIO_PIPE(n, _PCH_PP_OFF_DELAYS, _BXT_PP_OFF_DELAYS2) + +#define _PCH_DP_B 0xe4100 +#define PCH_DP_B _MMIO(_PCH_DP_B) +#define _PCH_DPB_AUX_CH_CTL 0xe4110 +#define _PCH_DPB_AUX_CH_DATA1 0xe4114 +#define _PCH_DPB_AUX_CH_DATA2 0xe4118 +#define _PCH_DPB_AUX_CH_DATA3 0xe411c +#define _PCH_DPB_AUX_CH_DATA4 0xe4120 +#define _PCH_DPB_AUX_CH_DATA5 0xe4124 + +#define _PCH_DP_C 0xe4200 +#define PCH_DP_C _MMIO(_PCH_DP_C) +#define _PCH_DPC_AUX_CH_CTL 0xe4210 +#define _PCH_DPC_AUX_CH_DATA1 0xe4214 +#define _PCH_DPC_AUX_CH_DATA2 0xe4218 +#define _PCH_DPC_AUX_CH_DATA3 0xe421c +#define _PCH_DPC_AUX_CH_DATA4 0xe4220 +#define _PCH_DPC_AUX_CH_DATA5 0xe4224 + +#define _PCH_DP_D 0xe4300 +#define PCH_DP_D _MMIO(_PCH_DP_D) +#define _PCH_DPD_AUX_CH_CTL 0xe4310 +#define _PCH_DPD_AUX_CH_DATA1 0xe4314 +#define _PCH_DPD_AUX_CH_DATA2 0xe4318 +#define _PCH_DPD_AUX_CH_DATA3 0xe431c +#define _PCH_DPD_AUX_CH_DATA4 0xe4320 +#define _PCH_DPD_AUX_CH_DATA5 0xe4324 + +#define PCH_DP_AUX_CH_CTL(port) _MMIO_PORT((port) - PORT_B, _PCH_DPB_AUX_CH_CTL, _PCH_DPC_AUX_CH_CTL) +#define PCH_DP_AUX_CH_DATA(port, i) _MMIO(_PORT((port) - PORT_B, _PCH_DPB_AUX_CH_DATA1, _PCH_DPC_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ /* CPT */ #define PORT_TRANS_A_SEL_CPT 0 @@ -6433,10 +6707,10 @@ enum skl_disp_power_wells { #define SDVO_PORT_TO_PIPE_CHV(val) (((val) & (3<<24)) >> 24) #define DP_PORT_TO_PIPE_CHV(val) (((val) & (3<<16)) >> 16) -#define TRANS_DP_CTL_A 0xe0300 -#define TRANS_DP_CTL_B 0xe1300 -#define TRANS_DP_CTL_C 0xe2300 -#define TRANS_DP_CTL(pipe) _PIPE(pipe, TRANS_DP_CTL_A, TRANS_DP_CTL_B) +#define _TRANS_DP_CTL_A 0xe0300 +#define _TRANS_DP_CTL_B 0xe1300 +#define _TRANS_DP_CTL_C 0xe2300 +#define TRANS_DP_CTL(pipe) _MMIO_PIPE(pipe, _TRANS_DP_CTL_A, _TRANS_DP_CTL_B) #define TRANS_DP_OUTPUT_ENABLE (1<<31) #define TRANS_DP_PORT_SEL_B (0<<29) #define TRANS_DP_PORT_SEL_C (1<<29) @@ -6489,40 +6763,40 @@ enum skl_disp_power_wells { #define EDP_LINK_TRAIN_VOL_EMP_MASK_IVB (0x3f<<22) -#define VLV_PMWGICZ 0x1300a4 +#define VLV_PMWGICZ _MMIO(0x1300a4) -#define FORCEWAKE 0xA18C -#define FORCEWAKE_VLV 0x1300b0 -#define FORCEWAKE_ACK_VLV 0x1300b4 -#define FORCEWAKE_MEDIA_VLV 0x1300b8 -#define FORCEWAKE_ACK_MEDIA_VLV 0x1300bc -#define FORCEWAKE_ACK_HSW 0x130044 -#define FORCEWAKE_ACK 0x130090 -#define VLV_GTLC_WAKE_CTRL 0x130090 +#define FORCEWAKE _MMIO(0xA18C) +#define FORCEWAKE_VLV _MMIO(0x1300b0) +#define FORCEWAKE_ACK_VLV _MMIO(0x1300b4) +#define FORCEWAKE_MEDIA_VLV _MMIO(0x1300b8) +#define FORCEWAKE_ACK_MEDIA_VLV _MMIO(0x1300bc) +#define FORCEWAKE_ACK_HSW _MMIO(0x130044) +#define FORCEWAKE_ACK _MMIO(0x130090) +#define VLV_GTLC_WAKE_CTRL _MMIO(0x130090) #define VLV_GTLC_RENDER_CTX_EXISTS (1 << 25) #define VLV_GTLC_MEDIA_CTX_EXISTS (1 << 24) #define VLV_GTLC_ALLOWWAKEREQ (1 << 0) -#define VLV_GTLC_PW_STATUS 0x130094 +#define VLV_GTLC_PW_STATUS _MMIO(0x130094) #define VLV_GTLC_ALLOWWAKEACK (1 << 0) #define VLV_GTLC_ALLOWWAKEERR (1 << 1) #define VLV_GTLC_PW_MEDIA_STATUS_MASK (1 << 5) #define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) -#define FORCEWAKE_MT 0xa188 /* multi-threaded */ -#define FORCEWAKE_MEDIA_GEN9 0xa270 -#define FORCEWAKE_RENDER_GEN9 0xa278 -#define FORCEWAKE_BLITTER_GEN9 0xa188 -#define FORCEWAKE_ACK_MEDIA_GEN9 0x0D88 -#define FORCEWAKE_ACK_RENDER_GEN9 0x0D84 -#define FORCEWAKE_ACK_BLITTER_GEN9 0x130044 +#define FORCEWAKE_MT _MMIO(0xa188) /* multi-threaded */ +#define FORCEWAKE_MEDIA_GEN9 _MMIO(0xa270) +#define FORCEWAKE_RENDER_GEN9 _MMIO(0xa278) +#define FORCEWAKE_BLITTER_GEN9 _MMIO(0xa188) +#define FORCEWAKE_ACK_MEDIA_GEN9 _MMIO(0x0D88) +#define FORCEWAKE_ACK_RENDER_GEN9 _MMIO(0x0D84) +#define FORCEWAKE_ACK_BLITTER_GEN9 _MMIO(0x130044) #define FORCEWAKE_KERNEL 0x1 #define FORCEWAKE_USER 0x2 -#define FORCEWAKE_MT_ACK 0x130040 -#define ECOBUS 0xa180 +#define FORCEWAKE_MT_ACK _MMIO(0x130040) +#define ECOBUS _MMIO(0xa180) #define FORCEWAKE_MT_ENABLE (1<<5) -#define VLV_SPAREG2H 0xA194 +#define VLV_SPAREG2H _MMIO(0xA194) -#define GTFIFODBG 0x120000 +#define GTFIFODBG _MMIO(0x120000) #define GT_FIFO_SBDROPERR (1<<6) #define GT_FIFO_BLOBDROPERR (1<<5) #define GT_FIFO_SB_READ_ABORTERR (1<<4) @@ -6531,23 +6805,23 @@ enum skl_disp_power_wells { #define GT_FIFO_IAWRERR (1<<1) #define GT_FIFO_IARDERR (1<<0) -#define GTFIFOCTL 0x120008 +#define GTFIFOCTL _MMIO(0x120008) #define GT_FIFO_FREE_ENTRIES_MASK 0x7f #define GT_FIFO_NUM_RESERVED_ENTRIES 20 #define GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL (1 << 12) #define GT_FIFO_CTL_RC6_POLICY_STALL (1 << 11) -#define HSW_IDICR 0x9008 +#define HSW_IDICR _MMIO(0x9008) #define IDIHASHMSK(x) (((x) & 0x3f) << 16) -#define HSW_EDRAM_PRESENT 0x120010 +#define HSW_EDRAM_PRESENT _MMIO(0x120010) #define EDRAM_ENABLED 0x1 -#define GEN6_UCGCTL1 0x9400 +#define GEN6_UCGCTL1 _MMIO(0x9400) # define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE (1 << 16) # define GEN6_BLBUNIT_CLOCK_GATE_DISABLE (1 << 5) # define GEN6_CSUNIT_CLOCK_GATE_DISABLE (1 << 7) -#define GEN6_UCGCTL2 0x9404 +#define GEN6_UCGCTL2 _MMIO(0x9404) # define GEN6_VFUNIT_CLOCK_GATE_DISABLE (1 << 31) # define GEN7_VDSUNIT_CLOCK_GATE_DISABLE (1 << 30) # define GEN7_TDLUNIT_CLOCK_GATE_DISABLE (1 << 22) @@ -6555,30 +6829,30 @@ enum skl_disp_power_wells { # define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE (1 << 12) # define GEN6_RCCUNIT_CLOCK_GATE_DISABLE (1 << 11) -#define GEN6_UCGCTL3 0x9408 +#define GEN6_UCGCTL3 _MMIO(0x9408) -#define GEN7_UCGCTL4 0x940c +#define GEN7_UCGCTL4 _MMIO(0x940c) #define GEN7_L3BANK2X_CLOCK_GATE_DISABLE (1<<25) -#define GEN6_RCGCTL1 0x9410 -#define GEN6_RCGCTL2 0x9414 -#define GEN6_RSTCTL 0x9420 +#define GEN6_RCGCTL1 _MMIO(0x9410) +#define GEN6_RCGCTL2 _MMIO(0x9414) +#define GEN6_RSTCTL _MMIO(0x9420) -#define GEN8_UCGCTL6 0x9430 +#define GEN8_UCGCTL6 _MMIO(0x9430) #define GEN8_GAPSUNIT_CLOCK_GATE_DISABLE (1<<24) #define GEN8_SDEUNIT_CLOCK_GATE_DISABLE (1<<14) #define GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ (1<<28) -#define GEN6_GFXPAUSE 0xA000 -#define GEN6_RPNSWREQ 0xA008 +#define GEN6_GFXPAUSE _MMIO(0xA000) +#define GEN6_RPNSWREQ _MMIO(0xA008) #define GEN6_TURBO_DISABLE (1<<31) #define GEN6_FREQUENCY(x) ((x)<<25) #define HSW_FREQUENCY(x) ((x)<<24) #define GEN9_FREQUENCY(x) ((x)<<23) #define GEN6_OFFSET(x) ((x)<<19) #define GEN6_AGGRESSIVE_TURBO (0<<15) -#define GEN6_RC_VIDEO_FREQ 0xA00C -#define GEN6_RC_CONTROL 0xA090 +#define GEN6_RC_VIDEO_FREQ _MMIO(0xA00C) +#define GEN6_RC_CONTROL _MMIO(0xA090) #define GEN6_RC_CTL_RC6pp_ENABLE (1<<16) #define GEN6_RC_CTL_RC6p_ENABLE (1<<17) #define GEN6_RC_CTL_RC6_ENABLE (1<<18) @@ -6588,16 +6862,16 @@ enum skl_disp_power_wells { #define GEN7_RC_CTL_TO_MODE (1<<28) #define GEN6_RC_CTL_EI_MODE(x) ((x)<<27) #define GEN6_RC_CTL_HW_ENABLE (1<<31) -#define GEN6_RP_DOWN_TIMEOUT 0xA010 -#define GEN6_RP_INTERRUPT_LIMITS 0xA014 -#define GEN6_RPSTAT1 0xA01C +#define GEN6_RP_DOWN_TIMEOUT _MMIO(0xA010) +#define GEN6_RP_INTERRUPT_LIMITS _MMIO(0xA014) +#define GEN6_RPSTAT1 _MMIO(0xA01C) #define GEN6_CAGF_SHIFT 8 #define HSW_CAGF_SHIFT 7 #define GEN9_CAGF_SHIFT 23 #define GEN6_CAGF_MASK (0x7f << GEN6_CAGF_SHIFT) #define HSW_CAGF_MASK (0x7f << HSW_CAGF_SHIFT) #define GEN9_CAGF_MASK (0x1ff << GEN9_CAGF_SHIFT) -#define GEN6_RP_CONTROL 0xA024 +#define GEN6_RP_CONTROL _MMIO(0xA024) #define GEN6_RP_MEDIA_TURBO (1<<11) #define GEN6_RP_MEDIA_MODE_MASK (3<<9) #define GEN6_RP_MEDIA_HW_TURBO_MODE (3<<9) @@ -6611,53 +6885,53 @@ enum skl_disp_power_wells { #define GEN6_RP_UP_BUSY_CONT (0x4<<3) #define GEN6_RP_DOWN_IDLE_AVG (0x2<<0) #define GEN6_RP_DOWN_IDLE_CONT (0x1<<0) -#define GEN6_RP_UP_THRESHOLD 0xA02C -#define GEN6_RP_DOWN_THRESHOLD 0xA030 -#define GEN6_RP_CUR_UP_EI 0xA050 +#define GEN6_RP_UP_THRESHOLD _MMIO(0xA02C) +#define GEN6_RP_DOWN_THRESHOLD _MMIO(0xA030) +#define GEN6_RP_CUR_UP_EI _MMIO(0xA050) #define GEN6_CURICONT_MASK 0xffffff -#define GEN6_RP_CUR_UP 0xA054 +#define GEN6_RP_CUR_UP _MMIO(0xA054) #define GEN6_CURBSYTAVG_MASK 0xffffff -#define GEN6_RP_PREV_UP 0xA058 -#define GEN6_RP_CUR_DOWN_EI 0xA05C +#define GEN6_RP_PREV_UP _MMIO(0xA058) +#define GEN6_RP_CUR_DOWN_EI _MMIO(0xA05C) #define GEN6_CURIAVG_MASK 0xffffff -#define GEN6_RP_CUR_DOWN 0xA060 -#define GEN6_RP_PREV_DOWN 0xA064 -#define GEN6_RP_UP_EI 0xA068 -#define GEN6_RP_DOWN_EI 0xA06C -#define GEN6_RP_IDLE_HYSTERSIS 0xA070 -#define GEN6_RPDEUHWTC 0xA080 -#define GEN6_RPDEUC 0xA084 -#define GEN6_RPDEUCSW 0xA088 -#define GEN6_RC_STATE 0xA094 -#define GEN6_RC1_WAKE_RATE_LIMIT 0xA098 -#define GEN6_RC6_WAKE_RATE_LIMIT 0xA09C -#define GEN6_RC6pp_WAKE_RATE_LIMIT 0xA0A0 -#define GEN6_RC_EVALUATION_INTERVAL 0xA0A8 -#define GEN6_RC_IDLE_HYSTERSIS 0xA0AC -#define GEN6_RC_SLEEP 0xA0B0 -#define GEN6_RCUBMABDTMR 0xA0B0 -#define GEN6_RC1e_THRESHOLD 0xA0B4 -#define GEN6_RC6_THRESHOLD 0xA0B8 -#define GEN6_RC6p_THRESHOLD 0xA0BC -#define VLV_RCEDATA 0xA0BC -#define GEN6_RC6pp_THRESHOLD 0xA0C0 -#define GEN6_PMINTRMSK 0xA168 +#define GEN6_RP_CUR_DOWN _MMIO(0xA060) +#define GEN6_RP_PREV_DOWN _MMIO(0xA064) +#define GEN6_RP_UP_EI _MMIO(0xA068) +#define GEN6_RP_DOWN_EI _MMIO(0xA06C) +#define GEN6_RP_IDLE_HYSTERSIS _MMIO(0xA070) +#define GEN6_RPDEUHWTC _MMIO(0xA080) +#define GEN6_RPDEUC _MMIO(0xA084) +#define GEN6_RPDEUCSW _MMIO(0xA088) +#define GEN6_RC_STATE _MMIO(0xA094) +#define GEN6_RC1_WAKE_RATE_LIMIT _MMIO(0xA098) +#define GEN6_RC6_WAKE_RATE_LIMIT _MMIO(0xA09C) +#define GEN6_RC6pp_WAKE_RATE_LIMIT _MMIO(0xA0A0) +#define GEN6_RC_EVALUATION_INTERVAL _MMIO(0xA0A8) +#define GEN6_RC_IDLE_HYSTERSIS _MMIO(0xA0AC) +#define GEN6_RC_SLEEP _MMIO(0xA0B0) +#define GEN6_RCUBMABDTMR _MMIO(0xA0B0) +#define GEN6_RC1e_THRESHOLD _MMIO(0xA0B4) +#define GEN6_RC6_THRESHOLD _MMIO(0xA0B8) +#define GEN6_RC6p_THRESHOLD _MMIO(0xA0BC) +#define VLV_RCEDATA _MMIO(0xA0BC) +#define GEN6_RC6pp_THRESHOLD _MMIO(0xA0C0) +#define GEN6_PMINTRMSK _MMIO(0xA168) #define GEN8_PMINTR_REDIRECT_TO_NON_DISP (1<<31) -#define VLV_PWRDWNUPCTL 0xA294 -#define GEN9_MEDIA_PG_IDLE_HYSTERESIS 0xA0C4 -#define GEN9_RENDER_PG_IDLE_HYSTERESIS 0xA0C8 -#define GEN9_PG_ENABLE 0xA210 +#define VLV_PWRDWNUPCTL _MMIO(0xA294) +#define GEN9_MEDIA_PG_IDLE_HYSTERESIS _MMIO(0xA0C4) +#define GEN9_RENDER_PG_IDLE_HYSTERESIS _MMIO(0xA0C8) +#define GEN9_PG_ENABLE _MMIO(0xA210) #define GEN9_RENDER_PG_ENABLE (1<<0) #define GEN9_MEDIA_PG_ENABLE (1<<1) -#define VLV_CHICKEN_3 (VLV_DISPLAY_BASE + 0x7040C) +#define VLV_CHICKEN_3 _MMIO(VLV_DISPLAY_BASE + 0x7040C) #define PIXEL_OVERLAP_CNT_MASK (3 << 30) #define PIXEL_OVERLAP_CNT_SHIFT 30 -#define GEN6_PMISR 0x44020 -#define GEN6_PMIMR 0x44024 /* rps_lock */ -#define GEN6_PMIIR 0x44028 -#define GEN6_PMIER 0x4402C +#define GEN6_PMISR _MMIO(0x44020) +#define GEN6_PMIMR _MMIO(0x44024) /* rps_lock */ +#define GEN6_PMIIR _MMIO(0x44028) +#define GEN6_PMIER _MMIO(0x4402C) #define GEN6_PM_MBOX_EVENT (1<<25) #define GEN6_PM_THERMAL_EVENT (1<<24) #define GEN6_PM_RP_DOWN_TIMEOUT (1<<6) @@ -6669,35 +6943,36 @@ enum skl_disp_power_wells { GEN6_PM_RP_DOWN_THRESHOLD | \ GEN6_PM_RP_DOWN_TIMEOUT) -#define GEN7_GT_SCRATCH_BASE 0x4F100 +#define GEN7_GT_SCRATCH(i) _MMIO(0x4F100 + (i) * 4) #define GEN7_GT_SCRATCH_REG_NUM 8 -#define VLV_GTLC_SURVIVABILITY_REG 0x130098 +#define VLV_GTLC_SURVIVABILITY_REG _MMIO(0x130098) #define VLV_GFX_CLK_STATUS_BIT (1<<3) #define VLV_GFX_CLK_FORCE_ON_BIT (1<<2) -#define GEN6_GT_GFX_RC6_LOCKED 0x138104 -#define VLV_COUNTER_CONTROL 0x138104 +#define GEN6_GT_GFX_RC6_LOCKED _MMIO(0x138104) +#define VLV_COUNTER_CONTROL _MMIO(0x138104) #define VLV_COUNT_RANGE_HIGH (1<<15) #define VLV_MEDIA_RC0_COUNT_EN (1<<5) #define VLV_RENDER_RC0_COUNT_EN (1<<4) #define VLV_MEDIA_RC6_COUNT_EN (1<<1) #define VLV_RENDER_RC6_COUNT_EN (1<<0) -#define GEN6_GT_GFX_RC6 0x138108 -#define VLV_GT_RENDER_RC6 0x138108 -#define VLV_GT_MEDIA_RC6 0x13810C +#define GEN6_GT_GFX_RC6 _MMIO(0x138108) +#define VLV_GT_RENDER_RC6 _MMIO(0x138108) +#define VLV_GT_MEDIA_RC6 _MMIO(0x13810C) -#define GEN6_GT_GFX_RC6p 0x13810C -#define GEN6_GT_GFX_RC6pp 0x138110 -#define VLV_RENDER_C0_COUNT 0x138118 -#define VLV_MEDIA_C0_COUNT 0x13811C +#define GEN6_GT_GFX_RC6p _MMIO(0x13810C) +#define GEN6_GT_GFX_RC6pp _MMIO(0x138110) +#define VLV_RENDER_C0_COUNT _MMIO(0x138118) +#define VLV_MEDIA_C0_COUNT _MMIO(0x13811C) -#define GEN6_PCODE_MAILBOX 0x138124 +#define GEN6_PCODE_MAILBOX _MMIO(0x138124) #define GEN6_PCODE_READY (1<<31) #define GEN6_PCODE_WRITE_RC6VIDS 0x4 #define GEN6_PCODE_READ_RC6VIDS 0x5 #define GEN6_ENCODE_RC6_VID(mv) (((mv) - 245) / 5) #define GEN6_DECODE_RC6_VID(vids) (((vids) * 5) + 245) +#define BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ 0x18 #define GEN9_PCODE_READ_MEM_LATENCY 0x6 #define GEN9_MEM_LATENCY_LEVEL_MASK 0xFF #define GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT 8 @@ -6714,12 +6989,12 @@ enum skl_disp_power_wells { #define HSW_PCODE_DE_WRITE_FREQ_REQ 0x17 #define DISPLAY_IPS_CONTROL 0x19 #define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A -#define GEN6_PCODE_DATA 0x138128 +#define GEN6_PCODE_DATA _MMIO(0x138128) #define GEN6_PCODE_FREQ_IA_RATIO_SHIFT 8 #define GEN6_PCODE_FREQ_RING_RATIO_SHIFT 16 -#define GEN6_PCODE_DATA1 0x13812C +#define GEN6_PCODE_DATA1 _MMIO(0x13812C) -#define GEN6_GT_CORE_STATUS 0x138060 +#define GEN6_GT_CORE_STATUS _MMIO(0x138060) #define GEN6_CORE_CPD_STATE_MASK (7<<4) #define GEN6_RCn_MASK 7 #define GEN6_RC0 0 @@ -6727,23 +7002,26 @@ enum skl_disp_power_wells { #define GEN6_RC6 3 #define GEN6_RC7 4 -#define CHV_POWER_SS0_SIG1 0xa720 -#define CHV_POWER_SS1_SIG1 0xa728 +#define GEN8_GT_SLICE_INFO _MMIO(0x138064) +#define GEN8_LSLICESTAT_MASK 0x7 + +#define CHV_POWER_SS0_SIG1 _MMIO(0xa720) +#define CHV_POWER_SS1_SIG1 _MMIO(0xa728) #define CHV_SS_PG_ENABLE (1<<1) #define CHV_EU08_PG_ENABLE (1<<9) #define CHV_EU19_PG_ENABLE (1<<17) #define CHV_EU210_PG_ENABLE (1<<25) -#define CHV_POWER_SS0_SIG2 0xa724 -#define CHV_POWER_SS1_SIG2 0xa72c +#define CHV_POWER_SS0_SIG2 _MMIO(0xa724) +#define CHV_POWER_SS1_SIG2 _MMIO(0xa72c) #define CHV_EU311_PG_ENABLE (1<<1) -#define GEN9_SLICE_PGCTL_ACK(slice) (0x804c + (slice)*0x4) +#define GEN9_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + (slice)*0x4) #define GEN9_PGCTL_SLICE_ACK (1 << 0) #define GEN9_PGCTL_SS_ACK(subslice) (1 << (2 + (subslice)*2)) -#define GEN9_SS01_EU_PGCTL_ACK(slice) (0x805c + (slice)*0x8) -#define GEN9_SS23_EU_PGCTL_ACK(slice) (0x8060 + (slice)*0x8) +#define GEN9_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + (slice)*0x8) +#define GEN9_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + (slice)*0x8) #define GEN9_PGCTL_SSA_EU08_ACK (1 << 0) #define GEN9_PGCTL_SSA_EU19_ACK (1 << 2) #define GEN9_PGCTL_SSA_EU210_ACK (1 << 4) @@ -6753,12 +7031,17 @@ enum skl_disp_power_wells { #define GEN9_PGCTL_SSB_EU210_ACK (1 << 12) #define GEN9_PGCTL_SSB_EU311_ACK (1 << 14) -#define GEN7_MISCCPCTL (0x9424) -#define GEN7_DOP_CLOCK_GATE_ENABLE (1<<0) +#define GEN7_MISCCPCTL _MMIO(0x9424) +#define GEN7_DOP_CLOCK_GATE_ENABLE (1<<0) +#define GEN8_DOP_CLOCK_GATE_CFCLK_ENABLE (1<<2) +#define GEN8_DOP_CLOCK_GATE_GUC_ENABLE (1<<4) +#define GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE (1<<6) + +#define GEN8_GARBCNTL _MMIO(0xB004) +#define GEN9_GAPS_TSV_CREDIT_DISABLE (1<<7) /* IVYBRIDGE DPF */ -#define GEN7_L3CDERRST1 0xB008 /* L3CD Error Status 1 */ -#define HSW_L3CDERRST11 0xB208 /* L3CD Error Status register 1 slice 1 */ +#define GEN7_L3CDERRST1(slice) _MMIO(0xB008 + (slice) * 0x200) /* L3CD Error Status 1 */ #define GEN7_L3CDERRST1_ROW_MASK (0x7ff<<14) #define GEN7_PARITY_ERROR_VALID (1<<13) #define GEN7_L3CDERRST1_BANK_MASK (3<<11) @@ -6771,116 +7054,102 @@ enum skl_disp_power_wells { ((reg & GEN7_L3CDERRST1_SUBBANK_MASK) >> 8) #define GEN7_L3CDERRST1_ENABLE (1<<7) -#define GEN7_L3LOG_BASE 0xB070 -#define HSW_L3LOG_BASE_SLICE1 0xB270 +#define GEN7_L3LOG(slice, i) _MMIO(0xB070 + (slice) * 0x200 + (i) * 4) #define GEN7_L3LOG_SIZE 0x80 -#define GEN7_HALF_SLICE_CHICKEN1 0xe100 /* IVB GT1 + VLV */ -#define GEN7_HALF_SLICE_CHICKEN1_GT2 0xf100 +#define GEN7_HALF_SLICE_CHICKEN1 _MMIO(0xe100) /* IVB GT1 + VLV */ +#define GEN7_HALF_SLICE_CHICKEN1_GT2 _MMIO(0xf100) #define GEN7_MAX_PS_THREAD_DEP (8<<12) #define GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE (1<<10) #define GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE (1<<4) #define GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE (1<<3) -#define GEN9_HALF_SLICE_CHICKEN5 0xe188 +#define GEN9_HALF_SLICE_CHICKEN5 _MMIO(0xe188) #define GEN9_DG_MIRROR_FIX_ENABLE (1<<5) #define GEN9_CCS_TLB_PREFETCH_ENABLE (1<<3) -#define GEN8_ROW_CHICKEN 0xe4f0 +#define GEN8_ROW_CHICKEN _MMIO(0xe4f0) #define PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE (1<<8) #define STALL_DOP_GATING_DISABLE (1<<5) -#define GEN7_ROW_CHICKEN2 0xe4f4 -#define GEN7_ROW_CHICKEN2_GT2 0xf4f4 +#define GEN7_ROW_CHICKEN2 _MMIO(0xe4f4) +#define GEN7_ROW_CHICKEN2_GT2 _MMIO(0xf4f4) #define DOP_CLOCK_GATING_DISABLE (1<<0) -#define HSW_ROW_CHICKEN3 0xe49c +#define HSW_ROW_CHICKEN3 _MMIO(0xe49c) #define HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE (1 << 6) -#define HALF_SLICE_CHICKEN3 0xe184 +#define HALF_SLICE_CHICKEN2 _MMIO(0xe180) +#define GEN8_ST_PO_DISABLE (1<<13) + +#define HALF_SLICE_CHICKEN3 _MMIO(0xe184) #define HSW_SAMPLE_C_PERFORMANCE (1<<9) #define GEN8_CENTROID_PIXEL_OPT_DIS (1<<8) #define GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC (1<<5) #define GEN8_SAMPLER_POWER_BYPASS_DIS (1<<1) -#define GEN9_HALF_SLICE_CHICKEN7 0xe194 +#define GEN9_HALF_SLICE_CHICKEN7 _MMIO(0xe194) #define GEN9_ENABLE_YV12_BUGFIX (1<<4) /* Audio */ -#define G4X_AUD_VID_DID (dev_priv->info.display_mmio_offset + 0x62020) +#define G4X_AUD_VID_DID _MMIO(dev_priv->info.display_mmio_offset + 0x62020) #define INTEL_AUDIO_DEVCL 0x808629FB #define INTEL_AUDIO_DEVBLC 0x80862801 #define INTEL_AUDIO_DEVCTG 0x80862802 -#define G4X_AUD_CNTL_ST 0x620B4 +#define G4X_AUD_CNTL_ST _MMIO(0x620B4) #define G4X_ELDV_DEVCL_DEVBLC (1 << 13) #define G4X_ELDV_DEVCTG (1 << 14) #define G4X_ELD_ADDR_MASK (0xf << 5) #define G4X_ELD_ACK (1 << 4) -#define G4X_HDMIW_HDMIEDID 0x6210C +#define G4X_HDMIW_HDMIEDID _MMIO(0x6210C) #define _IBX_HDMIW_HDMIEDID_A 0xE2050 #define _IBX_HDMIW_HDMIEDID_B 0xE2150 -#define IBX_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \ - _IBX_HDMIW_HDMIEDID_A, \ - _IBX_HDMIW_HDMIEDID_B) +#define IBX_HDMIW_HDMIEDID(pipe) _MMIO_PIPE(pipe, _IBX_HDMIW_HDMIEDID_A, \ + _IBX_HDMIW_HDMIEDID_B) #define _IBX_AUD_CNTL_ST_A 0xE20B4 #define _IBX_AUD_CNTL_ST_B 0xE21B4 -#define IBX_AUD_CNTL_ST(pipe) _PIPE(pipe, \ - _IBX_AUD_CNTL_ST_A, \ - _IBX_AUD_CNTL_ST_B) +#define IBX_AUD_CNTL_ST(pipe) _MMIO_PIPE(pipe, _IBX_AUD_CNTL_ST_A, \ + _IBX_AUD_CNTL_ST_B) #define IBX_ELD_BUFFER_SIZE_MASK (0x1f << 10) #define IBX_ELD_ADDRESS_MASK (0x1f << 5) #define IBX_ELD_ACK (1 << 4) -#define IBX_AUD_CNTL_ST2 0xE20C0 +#define IBX_AUD_CNTL_ST2 _MMIO(0xE20C0) #define IBX_CP_READY(port) ((1 << 1) << (((port) - 1) * 4)) #define IBX_ELD_VALID(port) ((1 << 0) << (((port) - 1) * 4)) #define _CPT_HDMIW_HDMIEDID_A 0xE5050 #define _CPT_HDMIW_HDMIEDID_B 0xE5150 -#define CPT_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \ - _CPT_HDMIW_HDMIEDID_A, \ - _CPT_HDMIW_HDMIEDID_B) +#define CPT_HDMIW_HDMIEDID(pipe) _MMIO_PIPE(pipe, _CPT_HDMIW_HDMIEDID_A, _CPT_HDMIW_HDMIEDID_B) #define _CPT_AUD_CNTL_ST_A 0xE50B4 #define _CPT_AUD_CNTL_ST_B 0xE51B4 -#define CPT_AUD_CNTL_ST(pipe) _PIPE(pipe, \ - _CPT_AUD_CNTL_ST_A, \ - _CPT_AUD_CNTL_ST_B) -#define CPT_AUD_CNTRL_ST2 0xE50C0 +#define CPT_AUD_CNTL_ST(pipe) _MMIO_PIPE(pipe, _CPT_AUD_CNTL_ST_A, _CPT_AUD_CNTL_ST_B) +#define CPT_AUD_CNTRL_ST2 _MMIO(0xE50C0) #define _VLV_HDMIW_HDMIEDID_A (VLV_DISPLAY_BASE + 0x62050) #define _VLV_HDMIW_HDMIEDID_B (VLV_DISPLAY_BASE + 0x62150) -#define VLV_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \ - _VLV_HDMIW_HDMIEDID_A, \ - _VLV_HDMIW_HDMIEDID_B) +#define VLV_HDMIW_HDMIEDID(pipe) _MMIO_PIPE(pipe, _VLV_HDMIW_HDMIEDID_A, _VLV_HDMIW_HDMIEDID_B) #define _VLV_AUD_CNTL_ST_A (VLV_DISPLAY_BASE + 0x620B4) #define _VLV_AUD_CNTL_ST_B (VLV_DISPLAY_BASE + 0x621B4) -#define VLV_AUD_CNTL_ST(pipe) _PIPE(pipe, \ - _VLV_AUD_CNTL_ST_A, \ - _VLV_AUD_CNTL_ST_B) -#define VLV_AUD_CNTL_ST2 (VLV_DISPLAY_BASE + 0x620C0) +#define VLV_AUD_CNTL_ST(pipe) _MMIO_PIPE(pipe, _VLV_AUD_CNTL_ST_A, _VLV_AUD_CNTL_ST_B) +#define VLV_AUD_CNTL_ST2 _MMIO(VLV_DISPLAY_BASE + 0x620C0) /* These are the 4 32-bit write offset registers for each stream * output buffer. It determines the offset from the * 3DSTATE_SO_BUFFERs that the next streamed vertex output goes to. */ -#define GEN7_SO_WRITE_OFFSET(n) (0x5280 + (n) * 4) +#define GEN7_SO_WRITE_OFFSET(n) _MMIO(0x5280 + (n) * 4) #define _IBX_AUD_CONFIG_A 0xe2000 #define _IBX_AUD_CONFIG_B 0xe2100 -#define IBX_AUD_CFG(pipe) _PIPE(pipe, \ - _IBX_AUD_CONFIG_A, \ - _IBX_AUD_CONFIG_B) +#define IBX_AUD_CFG(pipe) _MMIO_PIPE(pipe, _IBX_AUD_CONFIG_A, _IBX_AUD_CONFIG_B) #define _CPT_AUD_CONFIG_A 0xe5000 #define _CPT_AUD_CONFIG_B 0xe5100 -#define CPT_AUD_CFG(pipe) _PIPE(pipe, \ - _CPT_AUD_CONFIG_A, \ - _CPT_AUD_CONFIG_B) +#define CPT_AUD_CFG(pipe) _MMIO_PIPE(pipe, _CPT_AUD_CONFIG_A, _CPT_AUD_CONFIG_B) #define _VLV_AUD_CONFIG_A (VLV_DISPLAY_BASE + 0x62000) #define _VLV_AUD_CONFIG_B (VLV_DISPLAY_BASE + 0x62100) -#define VLV_AUD_CFG(pipe) _PIPE(pipe, \ - _VLV_AUD_CONFIG_A, \ - _VLV_AUD_CONFIG_B) +#define VLV_AUD_CFG(pipe) _MMIO_PIPE(pipe, _VLV_AUD_CONFIG_A, _VLV_AUD_CONFIG_B) #define AUD_CONFIG_N_VALUE_INDEX (1 << 29) #define AUD_CONFIG_N_PROG_ENABLE (1 << 28) @@ -6905,72 +7174,62 @@ enum skl_disp_power_wells { /* HSW Audio */ #define _HSW_AUD_CONFIG_A 0x65000 #define _HSW_AUD_CONFIG_B 0x65100 -#define HSW_AUD_CFG(pipe) _PIPE(pipe, \ - _HSW_AUD_CONFIG_A, \ - _HSW_AUD_CONFIG_B) +#define HSW_AUD_CFG(pipe) _MMIO_PIPE(pipe, _HSW_AUD_CONFIG_A, _HSW_AUD_CONFIG_B) #define _HSW_AUD_MISC_CTRL_A 0x65010 #define _HSW_AUD_MISC_CTRL_B 0x65110 -#define HSW_AUD_MISC_CTRL(pipe) _PIPE(pipe, \ - _HSW_AUD_MISC_CTRL_A, \ - _HSW_AUD_MISC_CTRL_B) +#define HSW_AUD_MISC_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_MISC_CTRL_A, _HSW_AUD_MISC_CTRL_B) #define _HSW_AUD_DIP_ELD_CTRL_ST_A 0x650b4 #define _HSW_AUD_DIP_ELD_CTRL_ST_B 0x651b4 -#define HSW_AUD_DIP_ELD_CTRL(pipe) _PIPE(pipe, \ - _HSW_AUD_DIP_ELD_CTRL_ST_A, \ - _HSW_AUD_DIP_ELD_CTRL_ST_B) +#define HSW_AUD_DIP_ELD_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_DIP_ELD_CTRL_ST_A, _HSW_AUD_DIP_ELD_CTRL_ST_B) /* Audio Digital Converter */ #define _HSW_AUD_DIG_CNVT_1 0x65080 #define _HSW_AUD_DIG_CNVT_2 0x65180 -#define AUD_DIG_CNVT(pipe) _PIPE(pipe, \ - _HSW_AUD_DIG_CNVT_1, \ - _HSW_AUD_DIG_CNVT_2) +#define AUD_DIG_CNVT(pipe) _MMIO_PIPE(pipe, _HSW_AUD_DIG_CNVT_1, _HSW_AUD_DIG_CNVT_2) #define DIP_PORT_SEL_MASK 0x3 #define _HSW_AUD_EDID_DATA_A 0x65050 #define _HSW_AUD_EDID_DATA_B 0x65150 -#define HSW_AUD_EDID_DATA(pipe) _PIPE(pipe, \ - _HSW_AUD_EDID_DATA_A, \ - _HSW_AUD_EDID_DATA_B) +#define HSW_AUD_EDID_DATA(pipe) _MMIO_PIPE(pipe, _HSW_AUD_EDID_DATA_A, _HSW_AUD_EDID_DATA_B) -#define HSW_AUD_PIPE_CONV_CFG 0x6507c -#define HSW_AUD_PIN_ELD_CP_VLD 0x650c0 +#define HSW_AUD_PIPE_CONV_CFG _MMIO(0x6507c) +#define HSW_AUD_PIN_ELD_CP_VLD _MMIO(0x650c0) #define AUDIO_INACTIVE(trans) ((1 << 3) << ((trans) * 4)) #define AUDIO_OUTPUT_ENABLE(trans) ((1 << 2) << ((trans) * 4)) #define AUDIO_CP_READY(trans) ((1 << 1) << ((trans) * 4)) #define AUDIO_ELD_VALID(trans) ((1 << 0) << ((trans) * 4)) -#define HSW_AUD_CHICKENBIT 0x65f10 +#define HSW_AUD_CHICKENBIT _MMIO(0x65f10) #define SKL_AUD_CODEC_WAKE_SIGNAL (1 << 15) /* HSW Power Wells */ -#define HSW_PWR_WELL_BIOS 0x45400 /* CTL1 */ -#define HSW_PWR_WELL_DRIVER 0x45404 /* CTL2 */ -#define HSW_PWR_WELL_KVMR 0x45408 /* CTL3 */ -#define HSW_PWR_WELL_DEBUG 0x4540C /* CTL4 */ +#define HSW_PWR_WELL_BIOS _MMIO(0x45400) /* CTL1 */ +#define HSW_PWR_WELL_DRIVER _MMIO(0x45404) /* CTL2 */ +#define HSW_PWR_WELL_KVMR _MMIO(0x45408) /* CTL3 */ +#define HSW_PWR_WELL_DEBUG _MMIO(0x4540C) /* CTL4 */ #define HSW_PWR_WELL_ENABLE_REQUEST (1<<31) #define HSW_PWR_WELL_STATE_ENABLED (1<<30) -#define HSW_PWR_WELL_CTL5 0x45410 +#define HSW_PWR_WELL_CTL5 _MMIO(0x45410) #define HSW_PWR_WELL_ENABLE_SINGLE_STEP (1<<31) #define HSW_PWR_WELL_PWR_GATE_OVERRIDE (1<<20) #define HSW_PWR_WELL_FORCE_ON (1<<19) -#define HSW_PWR_WELL_CTL6 0x45414 +#define HSW_PWR_WELL_CTL6 _MMIO(0x45414) /* SKL Fuse Status */ -#define SKL_FUSE_STATUS 0x42000 +#define SKL_FUSE_STATUS _MMIO(0x42000) #define SKL_FUSE_DOWNLOAD_STATUS (1<<31) #define SKL_FUSE_PG0_DIST_STATUS (1<<27) #define SKL_FUSE_PG1_DIST_STATUS (1<<26) #define SKL_FUSE_PG2_DIST_STATUS (1<<25) /* Per-pipe DDI Function Control */ -#define TRANS_DDI_FUNC_CTL_A 0x60400 -#define TRANS_DDI_FUNC_CTL_B 0x61400 -#define TRANS_DDI_FUNC_CTL_C 0x62400 -#define TRANS_DDI_FUNC_CTL_EDP 0x6F400 -#define TRANS_DDI_FUNC_CTL(tran) _TRANSCODER2(tran, TRANS_DDI_FUNC_CTL_A) +#define _TRANS_DDI_FUNC_CTL_A 0x60400 +#define _TRANS_DDI_FUNC_CTL_B 0x61400 +#define _TRANS_DDI_FUNC_CTL_C 0x62400 +#define _TRANS_DDI_FUNC_CTL_EDP 0x6F400 +#define TRANS_DDI_FUNC_CTL(tran) _MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL_A) #define TRANS_DDI_FUNC_ENABLE (1<<31) /* Those bits are ignored by pipe EDP since it can only connect to DDI A */ @@ -7000,9 +7259,9 @@ enum skl_disp_power_wells { #define TRANS_DDI_BFI_ENABLE (1<<4) /* DisplayPort Transport Control */ -#define DP_TP_CTL_A 0x64040 -#define DP_TP_CTL_B 0x64140 -#define DP_TP_CTL(port) _PORT(port, DP_TP_CTL_A, DP_TP_CTL_B) +#define _DP_TP_CTL_A 0x64040 +#define _DP_TP_CTL_B 0x64140 +#define DP_TP_CTL(port) _MMIO_PORT(port, _DP_TP_CTL_A, _DP_TP_CTL_B) #define DP_TP_CTL_ENABLE (1<<31) #define DP_TP_CTL_MODE_SST (0<<27) #define DP_TP_CTL_MODE_MST (1<<27) @@ -7018,9 +7277,9 @@ enum skl_disp_power_wells { #define DP_TP_CTL_SCRAMBLE_DISABLE (1<<7) /* DisplayPort Transport Status */ -#define DP_TP_STATUS_A 0x64044 -#define DP_TP_STATUS_B 0x64144 -#define DP_TP_STATUS(port) _PORT(port, DP_TP_STATUS_A, DP_TP_STATUS_B) +#define _DP_TP_STATUS_A 0x64044 +#define _DP_TP_STATUS_B 0x64144 +#define DP_TP_STATUS(port) _MMIO_PORT(port, _DP_TP_STATUS_A, _DP_TP_STATUS_B) #define DP_TP_STATUS_IDLE_DONE (1<<25) #define DP_TP_STATUS_ACT_SENT (1<<24) #define DP_TP_STATUS_MODE_STATUS_MST (1<<23) @@ -7030,9 +7289,9 @@ enum skl_disp_power_wells { #define DP_TP_STATUS_PAYLOAD_MAPPING_VC0 (3 << 0) /* DDI Buffer Control */ -#define DDI_BUF_CTL_A 0x64000 -#define DDI_BUF_CTL_B 0x64100 -#define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B) +#define _DDI_BUF_CTL_A 0x64000 +#define _DDI_BUF_CTL_B 0x64100 +#define DDI_BUF_CTL(port) _MMIO_PORT(port, _DDI_BUF_CTL_A, _DDI_BUF_CTL_B) #define DDI_BUF_CTL_ENABLE (1<<31) #define DDI_BUF_TRANS_SELECT(n) ((n) << 24) #define DDI_BUF_EMP_MASK (0xf<<24) @@ -7040,19 +7299,22 @@ enum skl_disp_power_wells { #define DDI_BUF_IS_IDLE (1<<7) #define DDI_A_4_LANES (1<<4) #define DDI_PORT_WIDTH(width) (((width) - 1) << 1) +#define DDI_PORT_WIDTH_MASK (7 << 1) +#define DDI_PORT_WIDTH_SHIFT 1 #define DDI_INIT_DISPLAY_DETECTED (1<<0) /* DDI Buffer Translations */ -#define DDI_BUF_TRANS_A 0x64E00 -#define DDI_BUF_TRANS_B 0x64E60 -#define DDI_BUF_TRANS(port) _PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B) +#define _DDI_BUF_TRANS_A 0x64E00 +#define _DDI_BUF_TRANS_B 0x64E60 +#define DDI_BUF_TRANS_LO(port, i) _MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8) +#define DDI_BUF_TRANS_HI(port, i) _MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8 + 4) /* Sideband Interface (SBI) is programmed indirectly, via * SBI_ADDR, which contains the register offset; and SBI_DATA, * which contains the payload */ -#define SBI_ADDR 0xC6000 -#define SBI_DATA 0xC6004 -#define SBI_CTL_STAT 0xC6008 +#define SBI_ADDR _MMIO(0xC6000) +#define SBI_DATA _MMIO(0xC6004) +#define SBI_CTL_STAT _MMIO(0xC6008) #define SBI_CTL_DEST_ICLK (0x0<<16) #define SBI_CTL_DEST_MPHY (0x1<<16) #define SBI_CTL_OP_IORD (0x2<<8) @@ -7083,12 +7345,12 @@ enum skl_disp_power_wells { #define SBI_GEN0_CFG_BUFFENABLE_DISABLE (1<<0) /* LPT PIXCLK_GATE */ -#define PIXCLK_GATE 0xC6020 +#define PIXCLK_GATE _MMIO(0xC6020) #define PIXCLK_GATE_UNGATE (1<<0) #define PIXCLK_GATE_GATE (0<<0) /* SPLL */ -#define SPLL_CTL 0x46020 +#define SPLL_CTL _MMIO(0x46020) #define SPLL_PLL_ENABLE (1<<31) #define SPLL_PLL_SSC (1<<28) #define SPLL_PLL_NON_SSC (2<<28) @@ -7100,9 +7362,9 @@ enum skl_disp_power_wells { #define SPLL_PLL_FREQ_MASK (3<<26) /* WRPLL */ -#define WRPLL_CTL1 0x46040 -#define WRPLL_CTL2 0x46060 -#define WRPLL_CTL(pll) (pll == 0 ? WRPLL_CTL1 : WRPLL_CTL2) +#define _WRPLL_CTL1 0x46040 +#define _WRPLL_CTL2 0x46060 +#define WRPLL_CTL(pll) _MMIO_PIPE(pll, _WRPLL_CTL1, _WRPLL_CTL2) #define WRPLL_PLL_ENABLE (1<<31) #define WRPLL_PLL_SSC (1<<28) #define WRPLL_PLL_NON_SSC (2<<28) @@ -7119,9 +7381,9 @@ enum skl_disp_power_wells { #define WRPLL_DIVIDER_FB_MASK (0xff<<16) /* Port clock selection */ -#define PORT_CLK_SEL_A 0x46100 -#define PORT_CLK_SEL_B 0x46104 -#define PORT_CLK_SEL(port) _PORT(port, PORT_CLK_SEL_A, PORT_CLK_SEL_B) +#define _PORT_CLK_SEL_A 0x46100 +#define _PORT_CLK_SEL_B 0x46104 +#define PORT_CLK_SEL(port) _MMIO_PORT(port, _PORT_CLK_SEL_A, _PORT_CLK_SEL_B) #define PORT_CLK_SEL_LCPLL_2700 (0<<29) #define PORT_CLK_SEL_LCPLL_1350 (1<<29) #define PORT_CLK_SEL_LCPLL_810 (2<<29) @@ -7133,18 +7395,18 @@ enum skl_disp_power_wells { #define PORT_CLK_SEL_MASK (7<<29) /* Transcoder clock selection */ -#define TRANS_CLK_SEL_A 0x46140 -#define TRANS_CLK_SEL_B 0x46144 -#define TRANS_CLK_SEL(tran) _TRANSCODER(tran, TRANS_CLK_SEL_A, TRANS_CLK_SEL_B) +#define _TRANS_CLK_SEL_A 0x46140 +#define _TRANS_CLK_SEL_B 0x46144 +#define TRANS_CLK_SEL(tran) _MMIO_TRANS(tran, _TRANS_CLK_SEL_A, _TRANS_CLK_SEL_B) /* For each transcoder, we need to select the corresponding port clock */ #define TRANS_CLK_SEL_DISABLED (0x0<<29) -#define TRANS_CLK_SEL_PORT(x) ((x+1)<<29) +#define TRANS_CLK_SEL_PORT(x) (((x)+1)<<29) -#define TRANSA_MSA_MISC 0x60410 -#define TRANSB_MSA_MISC 0x61410 -#define TRANSC_MSA_MISC 0x62410 -#define TRANS_EDP_MSA_MISC 0x6f410 -#define TRANS_MSA_MISC(tran) _TRANSCODER2(tran, TRANSA_MSA_MISC) +#define _TRANSA_MSA_MISC 0x60410 +#define _TRANSB_MSA_MISC 0x61410 +#define _TRANSC_MSA_MISC 0x62410 +#define _TRANS_EDP_MSA_MISC 0x6f410 +#define TRANS_MSA_MISC(tran) _MMIO_TRANS2(tran, _TRANSA_MSA_MISC) #define TRANS_MSA_SYNC_CLK (1<<0) #define TRANS_MSA_6_BPC (0<<5) @@ -7154,7 +7416,7 @@ enum skl_disp_power_wells { #define TRANS_MSA_16_BPC (4<<5) /* LCPLL Control */ -#define LCPLL_CTL 0x130040 +#define LCPLL_CTL _MMIO(0x130040) #define LCPLL_PLL_DISABLE (1<<31) #define LCPLL_PLL_LOCK (1<<30) #define LCPLL_CLK_FREQ_MASK (3<<26) @@ -7163,6 +7425,7 @@ enum skl_disp_power_wells { #define LCPLL_CLK_FREQ_337_5_BDW (2<<26) #define LCPLL_CLK_FREQ_675_BDW (3<<26) #define LCPLL_CD_CLOCK_DISABLE (1<<25) +#define LCPLL_ROOT_CD_CLOCK_DISABLE (1<<24) #define LCPLL_CD2X_CLOCK_DISABLE (1<<23) #define LCPLL_POWER_DOWN_ALLOW (1<<22) #define LCPLL_CD_SOURCE_FCLK (1<<21) @@ -7173,7 +7436,7 @@ enum skl_disp_power_wells { */ /* CDCLK_CTL */ -#define CDCLK_CTL 0x46000 +#define CDCLK_CTL _MMIO(0x46000) #define CDCLK_FREQ_SEL_MASK (3<<26) #define CDCLK_FREQ_450_432 (0<<26) #define CDCLK_FREQ_540 (1<<26) @@ -7189,12 +7452,12 @@ enum skl_disp_power_wells { #define BXT_CDCLK_SSA_PRECHARGE_ENABLE (1<<16) /* LCPLL_CTL */ -#define LCPLL1_CTL 0x46010 -#define LCPLL2_CTL 0x46014 +#define LCPLL1_CTL _MMIO(0x46010) +#define LCPLL2_CTL _MMIO(0x46014) #define LCPLL_PLL_ENABLE (1<<31) /* DPLL control1 */ -#define DPLL_CTRL1 0x6C058 +#define DPLL_CTRL1 _MMIO(0x6C058) #define DPLL_CTRL1_HDMI_MODE(id) (1<<((id)*6+5)) #define DPLL_CTRL1_SSC(id) (1<<((id)*6+4)) #define DPLL_CTRL1_LINK_RATE_MASK(id) (7<<((id)*6+1)) @@ -7209,104 +7472,99 @@ enum skl_disp_power_wells { #define DPLL_CTRL1_LINK_RATE_2160 5 /* DPLL control2 */ -#define DPLL_CTRL2 0x6C05C -#define DPLL_CTRL2_DDI_CLK_OFF(port) (1<<(port+15)) +#define DPLL_CTRL2 _MMIO(0x6C05C) +#define DPLL_CTRL2_DDI_CLK_OFF(port) (1<<((port)+15)) #define DPLL_CTRL2_DDI_CLK_SEL_MASK(port) (3<<((port)*3+1)) #define DPLL_CTRL2_DDI_CLK_SEL_SHIFT(port) ((port)*3+1) -#define DPLL_CTRL2_DDI_CLK_SEL(clk, port) (clk<<((port)*3+1)) +#define DPLL_CTRL2_DDI_CLK_SEL(clk, port) ((clk)<<((port)*3+1)) #define DPLL_CTRL2_DDI_SEL_OVERRIDE(port) (1<<((port)*3)) /* DPLL Status */ -#define DPLL_STATUS 0x6C060 +#define DPLL_STATUS _MMIO(0x6C060) #define DPLL_LOCK(id) (1<<((id)*8)) /* DPLL cfg */ -#define DPLL1_CFGCR1 0x6C040 -#define DPLL2_CFGCR1 0x6C048 -#define DPLL3_CFGCR1 0x6C050 +#define _DPLL1_CFGCR1 0x6C040 +#define _DPLL2_CFGCR1 0x6C048 +#define _DPLL3_CFGCR1 0x6C050 #define DPLL_CFGCR1_FREQ_ENABLE (1<<31) #define DPLL_CFGCR1_DCO_FRACTION_MASK (0x7fff<<9) -#define DPLL_CFGCR1_DCO_FRACTION(x) (x<<9) +#define DPLL_CFGCR1_DCO_FRACTION(x) ((x)<<9) #define DPLL_CFGCR1_DCO_INTEGER_MASK (0x1ff) -#define DPLL1_CFGCR2 0x6C044 -#define DPLL2_CFGCR2 0x6C04C -#define DPLL3_CFGCR2 0x6C054 +#define _DPLL1_CFGCR2 0x6C044 +#define _DPLL2_CFGCR2 0x6C04C +#define _DPLL3_CFGCR2 0x6C054 #define DPLL_CFGCR2_QDIV_RATIO_MASK (0xff<<8) -#define DPLL_CFGCR2_QDIV_RATIO(x) (x<<8) -#define DPLL_CFGCR2_QDIV_MODE(x) (x<<7) +#define DPLL_CFGCR2_QDIV_RATIO(x) ((x)<<8) +#define DPLL_CFGCR2_QDIV_MODE(x) ((x)<<7) #define DPLL_CFGCR2_KDIV_MASK (3<<5) -#define DPLL_CFGCR2_KDIV(x) (x<<5) +#define DPLL_CFGCR2_KDIV(x) ((x)<<5) #define DPLL_CFGCR2_KDIV_5 (0<<5) #define DPLL_CFGCR2_KDIV_2 (1<<5) #define DPLL_CFGCR2_KDIV_3 (2<<5) #define DPLL_CFGCR2_KDIV_1 (3<<5) #define DPLL_CFGCR2_PDIV_MASK (7<<2) -#define DPLL_CFGCR2_PDIV(x) (x<<2) +#define DPLL_CFGCR2_PDIV(x) ((x)<<2) #define DPLL_CFGCR2_PDIV_1 (0<<2) #define DPLL_CFGCR2_PDIV_2 (1<<2) #define DPLL_CFGCR2_PDIV_3 (2<<2) #define DPLL_CFGCR2_PDIV_7 (4<<2) #define DPLL_CFGCR2_CENTRAL_FREQ_MASK (3) -#define GET_CFG_CR1_REG(id) (DPLL1_CFGCR1 + (id - SKL_DPLL1) * 8) -#define GET_CFG_CR2_REG(id) (DPLL1_CFGCR2 + (id - SKL_DPLL1) * 8) +#define DPLL_CFGCR1(id) _MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR1, _DPLL2_CFGCR2) +#define DPLL_CFGCR2(id) _MMIO_PIPE((id) - SKL_DPLL1, _DPLL1_CFGCR2, _DPLL2_CFGCR2) /* BXT display engine PLL */ -#define BXT_DE_PLL_CTL 0x6d000 +#define BXT_DE_PLL_CTL _MMIO(0x6d000) #define BXT_DE_PLL_RATIO(x) (x) /* {60,65,100} * 19.2MHz */ #define BXT_DE_PLL_RATIO_MASK 0xff -#define BXT_DE_PLL_ENABLE 0x46070 +#define BXT_DE_PLL_ENABLE _MMIO(0x46070) #define BXT_DE_PLL_PLL_ENABLE (1 << 31) #define BXT_DE_PLL_LOCK (1 << 30) /* GEN9 DC */ -#define DC_STATE_EN 0x45504 +#define DC_STATE_EN _MMIO(0x45504) +#define DC_STATE_DISABLE 0 #define DC_STATE_EN_UPTO_DC5 (1<<0) #define DC_STATE_EN_DC9 (1<<3) - -/* -* SKL DC -*/ -#define DC_STATE_EN 0x45504 -#define DC_STATE_EN_UPTO_DC5 (1<<0) #define DC_STATE_EN_UPTO_DC6 (2<<0) #define DC_STATE_EN_UPTO_DC5_DC6_MASK 0x3 -#define DC_STATE_DEBUG 0x45520 +#define DC_STATE_DEBUG _MMIO(0x45520) #define DC_STATE_DEBUG_MASK_MEMORY_UP (1<<1) /* Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register, * since on HSW we can't write to it using I915_WRITE. */ -#define D_COMP_HSW (MCHBAR_MIRROR_BASE_SNB + 0x5F0C) -#define D_COMP_BDW 0x138144 +#define D_COMP_HSW _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5F0C) +#define D_COMP_BDW _MMIO(0x138144) #define D_COMP_RCOMP_IN_PROGRESS (1<<9) #define D_COMP_COMP_FORCE (1<<8) #define D_COMP_COMP_DISABLE (1<<0) /* Pipe WM_LINETIME - watermark line time */ -#define PIPE_WM_LINETIME_A 0x45270 -#define PIPE_WM_LINETIME_B 0x45274 -#define PIPE_WM_LINETIME(pipe) _PIPE(pipe, PIPE_WM_LINETIME_A, \ - PIPE_WM_LINETIME_B) +#define _PIPE_WM_LINETIME_A 0x45270 +#define _PIPE_WM_LINETIME_B 0x45274 +#define PIPE_WM_LINETIME(pipe) _MMIO_PIPE(pipe, _PIPE_WM_LINETIME_A, _PIPE_WM_LINETIME_B) #define PIPE_WM_LINETIME_MASK (0x1ff) #define PIPE_WM_LINETIME_TIME(x) ((x)) #define PIPE_WM_LINETIME_IPS_LINETIME_MASK (0x1ff<<16) #define PIPE_WM_LINETIME_IPS_LINETIME(x) ((x)<<16) /* SFUSE_STRAP */ -#define SFUSE_STRAP 0xc2014 +#define SFUSE_STRAP _MMIO(0xc2014) #define SFUSE_STRAP_FUSE_LOCK (1<<13) #define SFUSE_STRAP_DISPLAY_DISABLED (1<<7) +#define SFUSE_STRAP_CRT_DISABLED (1<<6) #define SFUSE_STRAP_DDIB_DETECTED (1<<2) #define SFUSE_STRAP_DDIC_DETECTED (1<<1) #define SFUSE_STRAP_DDID_DETECTED (1<<0) -#define WM_MISC 0x45260 +#define WM_MISC _MMIO(0x45260) #define WM_MISC_DATA_PARTITION_5_6 (1 << 0) -#define WM_DBG 0x45280 +#define WM_DBG _MMIO(0x45280) #define WM_DBG_DISALLOW_MULTIPLE_LP (1<<0) #define WM_DBG_DISALLOW_MAXFIFO (1<<1) #define WM_DBG_DISALLOW_SPRITE (1<<2) @@ -7343,27 +7601,134 @@ enum skl_disp_power_wells { #define _PIPE_B_CSC_POSTOFF_ME 0x49144 #define _PIPE_B_CSC_POSTOFF_LO 0x49148 -#define PIPE_CSC_COEFF_RY_GY(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RY_GY, _PIPE_B_CSC_COEFF_RY_GY) -#define PIPE_CSC_COEFF_BY(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BY, _PIPE_B_CSC_COEFF_BY) -#define PIPE_CSC_COEFF_RU_GU(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RU_GU, _PIPE_B_CSC_COEFF_RU_GU) -#define PIPE_CSC_COEFF_BU(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BU, _PIPE_B_CSC_COEFF_BU) -#define PIPE_CSC_COEFF_RV_GV(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_RV_GV, _PIPE_B_CSC_COEFF_RV_GV) -#define PIPE_CSC_COEFF_BV(pipe) _PIPE(pipe, _PIPE_A_CSC_COEFF_BV, _PIPE_B_CSC_COEFF_BV) -#define PIPE_CSC_MODE(pipe) _PIPE(pipe, _PIPE_A_CSC_MODE, _PIPE_B_CSC_MODE) -#define PIPE_CSC_PREOFF_HI(pipe) _PIPE(pipe, _PIPE_A_CSC_PREOFF_HI, _PIPE_B_CSC_PREOFF_HI) -#define PIPE_CSC_PREOFF_ME(pipe) _PIPE(pipe, _PIPE_A_CSC_PREOFF_ME, _PIPE_B_CSC_PREOFF_ME) -#define PIPE_CSC_PREOFF_LO(pipe) _PIPE(pipe, _PIPE_A_CSC_PREOFF_LO, _PIPE_B_CSC_PREOFF_LO) -#define PIPE_CSC_POSTOFF_HI(pipe) _PIPE(pipe, _PIPE_A_CSC_POSTOFF_HI, _PIPE_B_CSC_POSTOFF_HI) -#define PIPE_CSC_POSTOFF_ME(pipe) _PIPE(pipe, _PIPE_A_CSC_POSTOFF_ME, _PIPE_B_CSC_POSTOFF_ME) -#define PIPE_CSC_POSTOFF_LO(pipe) _PIPE(pipe, _PIPE_A_CSC_POSTOFF_LO, _PIPE_B_CSC_POSTOFF_LO) +#define PIPE_CSC_COEFF_RY_GY(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RY_GY, _PIPE_B_CSC_COEFF_RY_GY) +#define PIPE_CSC_COEFF_BY(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BY, _PIPE_B_CSC_COEFF_BY) +#define PIPE_CSC_COEFF_RU_GU(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RU_GU, _PIPE_B_CSC_COEFF_RU_GU) +#define PIPE_CSC_COEFF_BU(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BU, _PIPE_B_CSC_COEFF_BU) +#define PIPE_CSC_COEFF_RV_GV(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_RV_GV, _PIPE_B_CSC_COEFF_RV_GV) +#define PIPE_CSC_COEFF_BV(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_COEFF_BV, _PIPE_B_CSC_COEFF_BV) +#define PIPE_CSC_MODE(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_MODE, _PIPE_B_CSC_MODE) +#define PIPE_CSC_PREOFF_HI(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_HI, _PIPE_B_CSC_PREOFF_HI) +#define PIPE_CSC_PREOFF_ME(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_ME, _PIPE_B_CSC_PREOFF_ME) +#define PIPE_CSC_PREOFF_LO(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_PREOFF_LO, _PIPE_B_CSC_PREOFF_LO) +#define PIPE_CSC_POSTOFF_HI(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_HI, _PIPE_B_CSC_POSTOFF_HI) +#define PIPE_CSC_POSTOFF_ME(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_ME, _PIPE_B_CSC_POSTOFF_ME) +#define PIPE_CSC_POSTOFF_LO(pipe) _MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_LO, _PIPE_B_CSC_POSTOFF_LO) /* MIPI DSI registers */ #define _MIPI_PORT(port, a, c) _PORT3(port, a, 0, c) /* ports A and C only */ +#define _MMIO_MIPI(port, a, c) _MMIO(_MIPI_PORT(port, a, c)) + +/* BXT MIPI clock controls */ +#define BXT_MAX_VAR_OUTPUT_KHZ 39500 + +#define BXT_MIPI_CLOCK_CTL _MMIO(0x46090) +#define BXT_MIPI1_DIV_SHIFT 26 +#define BXT_MIPI2_DIV_SHIFT 10 +#define BXT_MIPI_DIV_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_DIV_SHIFT, \ + BXT_MIPI2_DIV_SHIFT) +/* Var clock divider to generate TX source. Result must be < 39.5 M */ +#define BXT_MIPI1_ESCLK_VAR_DIV_MASK (0x3F << 26) +#define BXT_MIPI2_ESCLK_VAR_DIV_MASK (0x3F << 10) +#define BXT_MIPI_ESCLK_VAR_DIV_MASK(port) \ + _MIPI_PORT(port, BXT_MIPI1_ESCLK_VAR_DIV_MASK, \ + BXT_MIPI2_ESCLK_VAR_DIV_MASK) + +#define BXT_MIPI_ESCLK_VAR_DIV(port, val) \ + (val << BXT_MIPI_DIV_SHIFT(port)) +/* TX control divider to select actual TX clock output from (8x/var) */ +#define BXT_MIPI1_TX_ESCLK_SHIFT 21 +#define BXT_MIPI2_TX_ESCLK_SHIFT 5 +#define BXT_MIPI_TX_ESCLK_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_SHIFT, \ + BXT_MIPI2_TX_ESCLK_SHIFT) +#define BXT_MIPI1_TX_ESCLK_FIXDIV_MASK (3 << 21) +#define BXT_MIPI2_TX_ESCLK_FIXDIV_MASK (3 << 5) +#define BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port) \ + _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_FIXDIV_MASK, \ + BXT_MIPI2_TX_ESCLK_FIXDIV_MASK) +#define BXT_MIPI_TX_ESCLK_8XDIV_BY2(port) \ + (0x0 << BXT_MIPI_TX_ESCLK_SHIFT(port)) +#define BXT_MIPI_TX_ESCLK_8XDIV_BY4(port) \ + (0x1 << BXT_MIPI_TX_ESCLK_SHIFT(port)) +#define BXT_MIPI_TX_ESCLK_8XDIV_BY8(port) \ + (0x2 << BXT_MIPI_TX_ESCLK_SHIFT(port)) +/* RX control divider to select actual RX clock output from 8x*/ +#define BXT_MIPI1_RX_ESCLK_SHIFT 19 +#define BXT_MIPI2_RX_ESCLK_SHIFT 3 +#define BXT_MIPI_RX_ESCLK_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_SHIFT, \ + BXT_MIPI2_RX_ESCLK_SHIFT) +#define BXT_MIPI1_RX_ESCLK_FIXDIV_MASK (3 << 19) +#define BXT_MIPI2_RX_ESCLK_FIXDIV_MASK (3 << 3) +#define BXT_MIPI_RX_ESCLK_FIXDIV_MASK(port) \ + (3 << BXT_MIPI_RX_ESCLK_SHIFT(port)) +#define BXT_MIPI_RX_ESCLK_8X_BY2(port) \ + (1 << BXT_MIPI_RX_ESCLK_SHIFT(port)) +#define BXT_MIPI_RX_ESCLK_8X_BY3(port) \ + (2 << BXT_MIPI_RX_ESCLK_SHIFT(port)) +#define BXT_MIPI_RX_ESCLK_8X_BY4(port) \ + (3 << BXT_MIPI_RX_ESCLK_SHIFT(port)) +/* BXT-A WA: Always prog DPHY dividers to 00 */ +#define BXT_MIPI1_DPHY_DIV_SHIFT 16 +#define BXT_MIPI2_DPHY_DIV_SHIFT 0 +#define BXT_MIPI_DPHY_DIV_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_DPHY_DIV_SHIFT, \ + BXT_MIPI2_DPHY_DIV_SHIFT) +#define BXT_MIPI_1_DPHY_DIVIDER_MASK (3 << 16) +#define BXT_MIPI_2_DPHY_DIVIDER_MASK (3 << 0) +#define BXT_MIPI_DPHY_DIVIDER_MASK(port) \ + (3 << BXT_MIPI_DPHY_DIV_SHIFT(port)) + +/* BXT MIPI mode configure */ +#define _BXT_MIPIA_TRANS_HACTIVE 0x6B0F8 +#define _BXT_MIPIC_TRANS_HACTIVE 0x6B8F8 +#define BXT_MIPI_TRANS_HACTIVE(tc) _MMIO_MIPI(tc, \ + _BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE) + +#define _BXT_MIPIA_TRANS_VACTIVE 0x6B0FC +#define _BXT_MIPIC_TRANS_VACTIVE 0x6B8FC +#define BXT_MIPI_TRANS_VACTIVE(tc) _MMIO_MIPI(tc, \ + _BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE) + +#define _BXT_MIPIA_TRANS_VTOTAL 0x6B100 +#define _BXT_MIPIC_TRANS_VTOTAL 0x6B900 +#define BXT_MIPI_TRANS_VTOTAL(tc) _MMIO_MIPI(tc, \ + _BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL) + +#define BXT_DSI_PLL_CTL _MMIO(0x161000) +#define BXT_DSI_PLL_PVD_RATIO_SHIFT 16 +#define BXT_DSI_PLL_PVD_RATIO_MASK (3 << BXT_DSI_PLL_PVD_RATIO_SHIFT) +#define BXT_DSI_PLL_PVD_RATIO_1 (1 << BXT_DSI_PLL_PVD_RATIO_SHIFT) +#define BXT_DSIC_16X_BY2 (1 << 10) +#define BXT_DSIC_16X_BY3 (2 << 10) +#define BXT_DSIC_16X_BY4 (3 << 10) +#define BXT_DSIA_16X_BY2 (1 << 8) +#define BXT_DSIA_16X_BY3 (2 << 8) +#define BXT_DSIA_16X_BY4 (3 << 8) +#define BXT_DSI_FREQ_SEL_SHIFT 8 +#define BXT_DSI_FREQ_SEL_MASK (0xF << BXT_DSI_FREQ_SEL_SHIFT) + +#define BXT_DSI_PLL_RATIO_MAX 0x7D +#define BXT_DSI_PLL_RATIO_MIN 0x22 +#define BXT_DSI_PLL_RATIO_MASK 0xFF +#define BXT_REF_CLOCK_KHZ 19200 + +#define BXT_DSI_PLL_ENABLE _MMIO(0x46080) +#define BXT_DSI_PLL_DO_ENABLE (1 << 31) +#define BXT_DSI_PLL_LOCKED (1 << 30) #define _MIPIA_PORT_CTRL (VLV_DISPLAY_BASE + 0x61190) #define _MIPIC_PORT_CTRL (VLV_DISPLAY_BASE + 0x61700) -#define MIPI_PORT_CTRL(port) _MIPI_PORT(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL) +#define MIPI_PORT_CTRL(port) _MMIO_MIPI(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL) + + /* BXT port control */ +#define _BXT_MIPIA_PORT_CTRL 0x6B0C0 +#define _BXT_MIPIC_PORT_CTRL 0x6B8C0 +#define BXT_MIPI_PORT_CTRL(tc) _MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL) + #define DPI_ENABLE (1 << 31) /* A + C */ #define MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT 27 #define MIPIA_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 27) @@ -7406,8 +7771,7 @@ enum skl_disp_power_wells { #define _MIPIA_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61194) #define _MIPIC_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61704) -#define MIPI_TEARING_CTRL(port) _MIPI_PORT(port, \ - _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL) +#define MIPI_TEARING_CTRL(port) _MMIO_MIPI(port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL) #define TEARING_EFFECT_DELAY_SHIFT 0 #define TEARING_EFFECT_DELAY_MASK (0xffff << 0) @@ -7418,8 +7782,7 @@ enum skl_disp_power_wells { #define _MIPIA_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb000) #define _MIPIC_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb800) -#define MIPI_DEVICE_READY(port) _MIPI_PORT(port, _MIPIA_DEVICE_READY, \ - _MIPIC_DEVICE_READY) +#define MIPI_DEVICE_READY(port) _MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY) #define BUS_POSSESSION (1 << 3) /* set to give bus to receiver */ #define ULPS_STATE_MASK (3 << 1) #define ULPS_STATE_ENTER (2 << 1) @@ -7429,12 +7792,10 @@ enum skl_disp_power_wells { #define _MIPIA_INTR_STAT (dev_priv->mipi_mmio_base + 0xb004) #define _MIPIC_INTR_STAT (dev_priv->mipi_mmio_base + 0xb804) -#define MIPI_INTR_STAT(port) _MIPI_PORT(port, _MIPIA_INTR_STAT, \ - _MIPIC_INTR_STAT) +#define MIPI_INTR_STAT(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT) #define _MIPIA_INTR_EN (dev_priv->mipi_mmio_base + 0xb008) #define _MIPIC_INTR_EN (dev_priv->mipi_mmio_base + 0xb808) -#define MIPI_INTR_EN(port) _MIPI_PORT(port, _MIPIA_INTR_EN, \ - _MIPIC_INTR_EN) +#define MIPI_INTR_EN(port) _MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN) #define TEARING_EFFECT (1 << 31) #define SPL_PKT_SENT_INTERRUPT (1 << 30) #define GEN_READ_DATA_AVAIL (1 << 29) @@ -7470,8 +7831,7 @@ enum skl_disp_power_wells { #define _MIPIA_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb00c) #define _MIPIC_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb80c) -#define MIPI_DSI_FUNC_PRG(port) _MIPI_PORT(port, _MIPIA_DSI_FUNC_PRG, \ - _MIPIC_DSI_FUNC_PRG) +#define MIPI_DSI_FUNC_PRG(port) _MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG) #define CMD_MODE_DATA_WIDTH_MASK (7 << 13) #define CMD_MODE_NOT_SUPPORTED (0 << 13) #define CMD_MODE_DATA_WIDTH_16_BIT (1 << 13) @@ -7494,32 +7854,27 @@ enum skl_disp_power_wells { #define _MIPIA_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb010) #define _MIPIC_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb810) -#define MIPI_HS_TX_TIMEOUT(port) _MIPI_PORT(port, _MIPIA_HS_TX_TIMEOUT, \ - _MIPIC_HS_TX_TIMEOUT) +#define MIPI_HS_TX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT) #define HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK 0xffffff #define _MIPIA_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb014) #define _MIPIC_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb814) -#define MIPI_LP_RX_TIMEOUT(port) _MIPI_PORT(port, _MIPIA_LP_RX_TIMEOUT, \ - _MIPIC_LP_RX_TIMEOUT) +#define MIPI_LP_RX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT) #define LOW_POWER_RX_TIMEOUT_COUNTER_MASK 0xffffff #define _MIPIA_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb018) #define _MIPIC_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb818) -#define MIPI_TURN_AROUND_TIMEOUT(port) _MIPI_PORT(port, \ - _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT) +#define MIPI_TURN_AROUND_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT) #define TURN_AROUND_TIMEOUT_MASK 0x3f #define _MIPIA_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb01c) #define _MIPIC_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb81c) -#define MIPI_DEVICE_RESET_TIMER(port) _MIPI_PORT(port, \ - _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER) +#define MIPI_DEVICE_RESET_TIMER(port) _MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER) #define DEVICE_RESET_TIMER_MASK 0xffff #define _MIPIA_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb020) #define _MIPIC_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb820) -#define MIPI_DPI_RESOLUTION(port) _MIPI_PORT(port, _MIPIA_DPI_RESOLUTION, \ - _MIPIC_DPI_RESOLUTION) +#define MIPI_DPI_RESOLUTION(port) _MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION) #define VERTICAL_ADDRESS_SHIFT 16 #define VERTICAL_ADDRESS_MASK (0xffff << 16) #define HORIZONTAL_ADDRESS_SHIFT 0 @@ -7527,8 +7882,7 @@ enum skl_disp_power_wells { #define _MIPIA_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb024) #define _MIPIC_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb824) -#define MIPI_DBI_FIFO_THROTTLE(port) _MIPI_PORT(port, \ - _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE) +#define MIPI_DBI_FIFO_THROTTLE(port) _MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE) #define DBI_FIFO_EMPTY_HALF (0 << 0) #define DBI_FIFO_EMPTY_QUARTER (1 << 0) #define DBI_FIFO_EMPTY_7_LOCATIONS (2 << 0) @@ -7536,50 +7890,41 @@ enum skl_disp_power_wells { /* regs below are bits 15:0 */ #define _MIPIA_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb028) #define _MIPIC_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb828) -#define MIPI_HSYNC_PADDING_COUNT(port) _MIPI_PORT(port, \ - _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT) +#define MIPI_HSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT) #define _MIPIA_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb02c) #define _MIPIC_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb82c) -#define MIPI_HBP_COUNT(port) _MIPI_PORT(port, _MIPIA_HBP_COUNT, \ - _MIPIC_HBP_COUNT) +#define MIPI_HBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT) #define _MIPIA_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb030) #define _MIPIC_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb830) -#define MIPI_HFP_COUNT(port) _MIPI_PORT(port, _MIPIA_HFP_COUNT, \ - _MIPIC_HFP_COUNT) +#define MIPI_HFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT) #define _MIPIA_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb034) #define _MIPIC_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb834) -#define MIPI_HACTIVE_AREA_COUNT(port) _MIPI_PORT(port, \ - _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT) +#define MIPI_HACTIVE_AREA_COUNT(port) _MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT) #define _MIPIA_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb038) #define _MIPIC_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb838) -#define MIPI_VSYNC_PADDING_COUNT(port) _MIPI_PORT(port, \ - _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT) +#define MIPI_VSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT) #define _MIPIA_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb03c) #define _MIPIC_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb83c) -#define MIPI_VBP_COUNT(port) _MIPI_PORT(port, _MIPIA_VBP_COUNT, \ - _MIPIC_VBP_COUNT) +#define MIPI_VBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT) #define _MIPIA_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb040) #define _MIPIC_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb840) -#define MIPI_VFP_COUNT(port) _MIPI_PORT(port, _MIPIA_VFP_COUNT, \ - _MIPIC_VFP_COUNT) +#define MIPI_VFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT) #define _MIPIA_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb044) #define _MIPIC_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb844) -#define MIPI_HIGH_LOW_SWITCH_COUNT(port) _MIPI_PORT(port, \ - _MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT) +#define MIPI_HIGH_LOW_SWITCH_COUNT(port) _MMIO_MIPI(port, _MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT) /* regs above are bits 15:0 */ #define _MIPIA_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb048) #define _MIPIC_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb848) -#define MIPI_DPI_CONTROL(port) _MIPI_PORT(port, _MIPIA_DPI_CONTROL, \ - _MIPIC_DPI_CONTROL) +#define MIPI_DPI_CONTROL(port) _MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL) #define DPI_LP_MODE (1 << 6) #define BACKLIGHT_OFF (1 << 5) #define BACKLIGHT_ON (1 << 4) @@ -7590,29 +7935,26 @@ enum skl_disp_power_wells { #define _MIPIA_DPI_DATA (dev_priv->mipi_mmio_base + 0xb04c) #define _MIPIC_DPI_DATA (dev_priv->mipi_mmio_base + 0xb84c) -#define MIPI_DPI_DATA(port) _MIPI_PORT(port, _MIPIA_DPI_DATA, \ - _MIPIC_DPI_DATA) +#define MIPI_DPI_DATA(port) _MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA) #define COMMAND_BYTE_SHIFT 0 #define COMMAND_BYTE_MASK (0x3f << 0) #define _MIPIA_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb050) #define _MIPIC_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb850) -#define MIPI_INIT_COUNT(port) _MIPI_PORT(port, _MIPIA_INIT_COUNT, \ - _MIPIC_INIT_COUNT) +#define MIPI_INIT_COUNT(port) _MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT) #define MASTER_INIT_TIMER_SHIFT 0 #define MASTER_INIT_TIMER_MASK (0xffff << 0) #define _MIPIA_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb054) #define _MIPIC_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb854) -#define MIPI_MAX_RETURN_PKT_SIZE(port) _MIPI_PORT(port, \ +#define MIPI_MAX_RETURN_PKT_SIZE(port) _MMIO_MIPI(port, \ _MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE) #define MAX_RETURN_PKT_SIZE_SHIFT 0 #define MAX_RETURN_PKT_SIZE_MASK (0x3ff << 0) #define _MIPIA_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb058) #define _MIPIC_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb858) -#define MIPI_VIDEO_MODE_FORMAT(port) _MIPI_PORT(port, \ - _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT) +#define MIPI_VIDEO_MODE_FORMAT(port) _MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT) #define RANDOM_DPI_DISPLAY_RESOLUTION (1 << 4) #define DISABLE_VIDEO_BTA (1 << 3) #define IP_TG_CONFIG (1 << 2) @@ -7622,8 +7964,7 @@ enum skl_disp_power_wells { #define _MIPIA_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb05c) #define _MIPIC_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb85c) -#define MIPI_EOT_DISABLE(port) _MIPI_PORT(port, _MIPIA_EOT_DISABLE, \ - _MIPIC_EOT_DISABLE) +#define MIPI_EOT_DISABLE(port) _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE) #define LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 7) #define HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 6) #define LOW_CONTENTION_RECOVERY_DISABLE (1 << 5) @@ -7635,31 +7976,26 @@ enum skl_disp_power_wells { #define _MIPIA_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb060) #define _MIPIC_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb860) -#define MIPI_LP_BYTECLK(port) _MIPI_PORT(port, _MIPIA_LP_BYTECLK, \ - _MIPIC_LP_BYTECLK) +#define MIPI_LP_BYTECLK(port) _MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK) #define LP_BYTECLK_SHIFT 0 #define LP_BYTECLK_MASK (0xffff << 0) /* bits 31:0 */ #define _MIPIA_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb064) #define _MIPIC_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb864) -#define MIPI_LP_GEN_DATA(port) _MIPI_PORT(port, _MIPIA_LP_GEN_DATA, \ - _MIPIC_LP_GEN_DATA) +#define MIPI_LP_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA) /* bits 31:0 */ #define _MIPIA_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb068) #define _MIPIC_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb868) -#define MIPI_HS_GEN_DATA(port) _MIPI_PORT(port, _MIPIA_HS_GEN_DATA, \ - _MIPIC_HS_GEN_DATA) +#define MIPI_HS_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA) #define _MIPIA_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb06c) #define _MIPIC_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb86c) -#define MIPI_LP_GEN_CTRL(port) _MIPI_PORT(port, _MIPIA_LP_GEN_CTRL, \ - _MIPIC_LP_GEN_CTRL) +#define MIPI_LP_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL) #define _MIPIA_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb070) #define _MIPIC_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb870) -#define MIPI_HS_GEN_CTRL(port) _MIPI_PORT(port, _MIPIA_HS_GEN_CTRL, \ - _MIPIC_HS_GEN_CTRL) +#define MIPI_HS_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL) #define LONG_PACKET_WORD_COUNT_SHIFT 8 #define LONG_PACKET_WORD_COUNT_MASK (0xffff << 8) #define SHORT_PACKET_PARAM_SHIFT 8 @@ -7667,13 +8003,12 @@ enum skl_disp_power_wells { #define VIRTUAL_CHANNEL_SHIFT 6 #define VIRTUAL_CHANNEL_MASK (3 << 6) #define DATA_TYPE_SHIFT 0 -#define DATA_TYPE_MASK (3f << 0) +#define DATA_TYPE_MASK (0x3f << 0) /* data type values, see include/video/mipi_display.h */ #define _MIPIA_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb074) #define _MIPIC_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb874) -#define MIPI_GEN_FIFO_STAT(port) _MIPI_PORT(port, _MIPIA_GEN_FIFO_STAT, \ - _MIPIC_GEN_FIFO_STAT) +#define MIPI_GEN_FIFO_STAT(port) _MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT) #define DPI_FIFO_EMPTY (1 << 28) #define DBI_FIFO_EMPTY (1 << 27) #define LP_CTRL_FIFO_EMPTY (1 << 26) @@ -7691,16 +8026,14 @@ enum skl_disp_power_wells { #define _MIPIA_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb078) #define _MIPIC_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb878) -#define MIPI_HS_LP_DBI_ENABLE(port) _MIPI_PORT(port, \ - _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE) +#define MIPI_HS_LP_DBI_ENABLE(port) _MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE) #define DBI_HS_LP_MODE_MASK (1 << 0) #define DBI_LP_MODE (1 << 0) #define DBI_HS_MODE (0 << 0) #define _MIPIA_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb080) #define _MIPIC_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb880) -#define MIPI_DPHY_PARAM(port) _MIPI_PORT(port, _MIPIA_DPHY_PARAM, \ - _MIPIC_DPHY_PARAM) +#define MIPI_DPHY_PARAM(port) _MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM) #define EXIT_ZERO_COUNT_SHIFT 24 #define EXIT_ZERO_COUNT_MASK (0x3f << 24) #define TRAIL_COUNT_SHIFT 16 @@ -7713,15 +8046,11 @@ enum skl_disp_power_wells { /* bits 31:0 */ #define _MIPIA_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb084) #define _MIPIC_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb884) -#define MIPI_DBI_BW_CTRL(port) _MIPI_PORT(port, _MIPIA_DBI_BW_CTRL, \ - _MIPIC_DBI_BW_CTRL) - -#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base \ - + 0xb088) -#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base \ - + 0xb888) -#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port) _MIPI_PORT(port, \ - _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT) +#define MIPI_DBI_BW_CTRL(port) _MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL) + +#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb088) +#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb888) +#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT) #define LP_HS_SSW_CNT_SHIFT 16 #define LP_HS_SSW_CNT_MASK (0xffff << 16) #define HS_LP_PWR_SW_CNT_SHIFT 0 @@ -7729,19 +8058,16 @@ enum skl_disp_power_wells { #define _MIPIA_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb08c) #define _MIPIC_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb88c) -#define MIPI_STOP_STATE_STALL(port) _MIPI_PORT(port, \ - _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL) +#define MIPI_STOP_STATE_STALL(port) _MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL) #define STOP_STATE_STALL_COUNTER_SHIFT 0 #define STOP_STATE_STALL_COUNTER_MASK (0xff << 0) #define _MIPIA_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb090) #define _MIPIC_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb890) -#define MIPI_INTR_STAT_REG_1(port) _MIPI_PORT(port, \ - _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1) +#define MIPI_INTR_STAT_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1) #define _MIPIA_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb094) #define _MIPIC_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb894) -#define MIPI_INTR_EN_REG_1(port) _MIPI_PORT(port, _MIPIA_INTR_EN_REG_1, \ - _MIPIC_INTR_EN_REG_1) +#define MIPI_INTR_EN_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1) #define RX_CONTENTION_DETECTED (1 << 0) /* XXX: only pipe A ?!? */ @@ -7761,8 +8087,7 @@ enum skl_disp_power_wells { #define _MIPIA_CTRL (dev_priv->mipi_mmio_base + 0xb104) #define _MIPIC_CTRL (dev_priv->mipi_mmio_base + 0xb904) -#define MIPI_CTRL(port) _MIPI_PORT(port, _MIPIA_CTRL, \ - _MIPIC_CTRL) +#define MIPI_CTRL(port) _MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL) #define ESCAPE_CLOCK_DIVIDER_SHIFT 5 /* A only */ #define ESCAPE_CLOCK_DIVIDER_MASK (3 << 5) #define ESCAPE_CLOCK_DIVIDER_1 (0 << 5) @@ -7774,25 +8099,27 @@ enum skl_disp_power_wells { #define READ_REQUEST_PRIORITY_HIGH (3 << 3) #define RGB_FLIP_TO_BGR (1 << 2) +#define BXT_PIPE_SELECT_MASK (7 << 7) +#define BXT_PIPE_SELECT_C (2 << 7) +#define BXT_PIPE_SELECT_B (1 << 7) +#define BXT_PIPE_SELECT_A (0 << 7) + #define _MIPIA_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb108) #define _MIPIC_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb908) -#define MIPI_DATA_ADDRESS(port) _MIPI_PORT(port, _MIPIA_DATA_ADDRESS, \ - _MIPIC_DATA_ADDRESS) +#define MIPI_DATA_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS) #define DATA_MEM_ADDRESS_SHIFT 5 #define DATA_MEM_ADDRESS_MASK (0x7ffffff << 5) #define DATA_VALID (1 << 0) #define _MIPIA_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb10c) #define _MIPIC_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb90c) -#define MIPI_DATA_LENGTH(port) _MIPI_PORT(port, _MIPIA_DATA_LENGTH, \ - _MIPIC_DATA_LENGTH) +#define MIPI_DATA_LENGTH(port) _MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH) #define DATA_LENGTH_SHIFT 0 #define DATA_LENGTH_MASK (0xfffff << 0) #define _MIPIA_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb110) #define _MIPIC_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb910) -#define MIPI_COMMAND_ADDRESS(port) _MIPI_PORT(port, \ - _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS) +#define MIPI_COMMAND_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS) #define COMMAND_MEM_ADDRESS_SHIFT 5 #define COMMAND_MEM_ADDRESS_MASK (0x7ffffff << 5) #define AUTO_PWG_ENABLE (1 << 2) @@ -7801,25 +8128,30 @@ enum skl_disp_power_wells { #define _MIPIA_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb114) #define _MIPIC_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb914) -#define MIPI_COMMAND_LENGTH(port) _MIPI_PORT(port, _MIPIA_COMMAND_LENGTH, \ - _MIPIC_COMMAND_LENGTH) +#define MIPI_COMMAND_LENGTH(port) _MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH) #define COMMAND_LENGTH_SHIFT(n) (8 * (n)) /* n: 0...3 */ #define COMMAND_LENGTH_MASK(n) (0xff << (8 * (n))) #define _MIPIA_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb118) #define _MIPIC_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb918) -#define MIPI_READ_DATA_RETURN(port, n) \ - (_MIPI_PORT(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) \ - + 4 * (n)) /* n: 0...7 */ +#define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */ #define _MIPIA_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb138) #define _MIPIC_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb938) -#define MIPI_READ_DATA_VALID(port) _MIPI_PORT(port, \ - _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) +#define MIPI_READ_DATA_VALID(port) _MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) #define READ_DATA_VALID(n) (1 << (n)) /* For UMS only (deprecated): */ #define _PALETTE_A (dev_priv->info.display_mmio_offset + 0xa000) #define _PALETTE_B (dev_priv->info.display_mmio_offset + 0xa800) +/* MOCS (Memory Object Control State) registers */ +#define GEN9_LNCFCMOCS(i) _MMIO(0xb020 + (i) * 4) /* L3 Cache Control */ + +#define GEN9_GFX_MOCS(i) _MMIO(0xc800 + (i) * 4) /* Graphics MOCS registers */ +#define GEN9_MFX0_MOCS(i) _MMIO(0xc900 + (i) * 4) /* Media 0 MOCS registers */ +#define GEN9_MFX1_MOCS(i) _MMIO(0xca00 + (i) * 4) /* Media 1 MOCS registers */ +#define GEN9_VEBOX_MOCS(i) _MMIO(0xcb00 + (i) * 4) /* Video MOCS registers */ +#define GEN9_BLT_MOCS(i) _MMIO(0xcc00 + (i) * 4) /* Blitter MOCS registers */ + #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index cf67f82f7b7f..2d9182189422 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -92,7 +92,7 @@ static void i915_restore_display(struct drm_device *dev) } /* only restore FBC info on the platform that supports FBC*/ - intel_fbc_disable(dev); + intel_fbc_disable(dev_priv); /* restore FBC interval */ if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev)) @@ -122,12 +122,24 @@ int i915_save_state(struct drm_device *dev) dev_priv->regfile.saveMI_ARB_STATE = I915_READ(MI_ARB_STATE); /* Scratch space */ - for (i = 0; i < 16; i++) { - dev_priv->regfile.saveSWF0[i] = I915_READ(SWF00 + (i << 2)); - dev_priv->regfile.saveSWF1[i] = I915_READ(SWF10 + (i << 2)); + if (IS_GEN2(dev_priv) && IS_MOBILE(dev_priv)) { + for (i = 0; i < 7; i++) { + dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i)); + dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); + } + for (i = 0; i < 3; i++) + dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i)); + } else if (IS_GEN2(dev_priv)) { + for (i = 0; i < 7; i++) + dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); + } else if (HAS_GMCH_DISPLAY(dev_priv)) { + for (i = 0; i < 16; i++) { + dev_priv->regfile.saveSWF0[i] = I915_READ(SWF0(i)); + dev_priv->regfile.saveSWF1[i] = I915_READ(SWF1(i)); + } + for (i = 0; i < 3; i++) + dev_priv->regfile.saveSWF3[i] = I915_READ(SWF3(i)); } - for (i = 0; i < 3; i++) - dev_priv->regfile.saveSWF2[i] = I915_READ(SWF30 + (i << 2)); mutex_unlock(&dev->struct_mutex); @@ -156,12 +168,25 @@ int i915_restore_state(struct drm_device *dev) /* Memory arbitration state */ I915_WRITE(MI_ARB_STATE, dev_priv->regfile.saveMI_ARB_STATE | 0xffff0000); - for (i = 0; i < 16; i++) { - I915_WRITE(SWF00 + (i << 2), dev_priv->regfile.saveSWF0[i]); - I915_WRITE(SWF10 + (i << 2), dev_priv->regfile.saveSWF1[i]); + /* Scratch space */ + if (IS_GEN2(dev_priv) && IS_MOBILE(dev_priv)) { + for (i = 0; i < 7; i++) { + I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]); + I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); + } + for (i = 0; i < 3; i++) + I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]); + } else if (IS_GEN2(dev_priv)) { + for (i = 0; i < 7; i++) + I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); + } else if (HAS_GMCH_DISPLAY(dev_priv)) { + for (i = 0; i < 16; i++) { + I915_WRITE(SWF0(i), dev_priv->regfile.saveSWF0[i]); + I915_WRITE(SWF1(i), dev_priv->regfile.saveSWF1[i]); + } + for (i = 0; i < 3; i++) + I915_WRITE(SWF3(i), dev_priv->regfile.saveSWF3[i]); } - for (i = 0; i < 3; i++) - I915_WRITE(SWF30 + (i << 2), dev_priv->regfile.saveSWF2[i]); mutex_unlock(&dev->struct_mutex); diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 247626885f49..f929c61f0fa2 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -35,11 +35,12 @@ #define dev_to_drm_minor(d) dev_get_drvdata((d)) #ifdef CONFIG_PM -static u32 calc_residency(struct drm_device *dev, const u32 reg) +static u32 calc_residency(struct drm_device *dev, + i915_reg_t reg) { struct drm_i915_private *dev_priv = dev->dev_private; u64 raw_time; /* 32b value may overflow during fixed point math */ - u64 units = 128ULL, div = 100000ULL, bias = 100ULL; + u64 units = 128ULL, div = 100000ULL; u32 ret; if (!intel_enable_rc6(dev)) @@ -49,49 +50,19 @@ static u32 calc_residency(struct drm_device *dev, const u32 reg) /* On VLV and CHV, residency time is in CZ units rather than 1.28us */ if (IS_VALLEYVIEW(dev)) { - u32 clk_reg, czcount_30ns; - - if (IS_CHERRYVIEW(dev)) - clk_reg = CHV_CLK_CTL1; - else - clk_reg = VLV_CLK_CTL2; - - czcount_30ns = I915_READ(clk_reg) >> CLK_CTL2_CZCOUNT_30NS_SHIFT; - - if (!czcount_30ns) { - WARN(!czcount_30ns, "bogus CZ count value"); - ret = 0; - goto out; - } - - units = 0; - div = 1000000ULL; - - if (IS_CHERRYVIEW(dev)) { - /* Special case for 320Mhz */ - if (czcount_30ns == 1) { - div = 10000000ULL; - units = 3125ULL; - } else { - /* chv counts are one less */ - czcount_30ns += 1; - } - } - - if (units == 0) - units = DIV_ROUND_UP_ULL(30ULL * bias, - (u64)czcount_30ns); + units = 1; + div = dev_priv->czclk_freq; if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH) units <<= 8; - - div = div * bias; + } else if (IS_BROXTON(dev)) { + units = 1; + div = 1200; /* 833.33ns */ } raw_time = I915_READ(reg) * units; ret = DIV_ROUND_UP_ULL(raw_time, div); -out: intel_runtime_pm_put(dev_priv); return ret; } diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 849a2590e010..52b2d409945d 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -17,8 +17,8 @@ /* pipe updates */ TRACE_EVENT(i915_pipe_update_start, - TP_PROTO(struct intel_crtc *crtc, u32 min, u32 max), - TP_ARGS(crtc, min, max), + TP_PROTO(struct intel_crtc *crtc), + TP_ARGS(crtc), TP_STRUCT__entry( __field(enum pipe, pipe) @@ -33,8 +33,8 @@ TRACE_EVENT(i915_pipe_update_start, __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, crtc->pipe); __entry->scanline = intel_get_crtc_scanline(crtc); - __entry->min = min; - __entry->max = max; + __entry->min = crtc->debug.min_vbl; + __entry->max = crtc->debug.max_vbl; ), TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u", @@ -43,8 +43,8 @@ TRACE_EVENT(i915_pipe_update_start, ); TRACE_EVENT(i915_pipe_update_vblank_evaded, - TP_PROTO(struct intel_crtc *crtc, u32 min, u32 max, u32 frame), - TP_ARGS(crtc, min, max, frame), + TP_PROTO(struct intel_crtc *crtc), + TP_ARGS(crtc), TP_STRUCT__entry( __field(enum pipe, pipe) @@ -56,10 +56,10 @@ TRACE_EVENT(i915_pipe_update_vblank_evaded, TP_fast_assign( __entry->pipe = crtc->pipe; - __entry->frame = frame; - __entry->scanline = intel_get_crtc_scanline(crtc); - __entry->min = min; - __entry->max = max; + __entry->frame = crtc->debug.start_vbl_count; + __entry->scanline = crtc->debug.scanline_start; + __entry->min = crtc->debug.min_vbl; + __entry->max = crtc->debug.max_vbl; ), TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u", @@ -68,8 +68,8 @@ TRACE_EVENT(i915_pipe_update_vblank_evaded, ); TRACE_EVENT(i915_pipe_update_end, - TP_PROTO(struct intel_crtc *crtc, u32 frame), - TP_ARGS(crtc, frame), + TP_PROTO(struct intel_crtc *crtc, u32 frame, int scanline_end), + TP_ARGS(crtc, frame, scanline_end), TP_STRUCT__entry( __field(enum pipe, pipe) @@ -80,7 +80,7 @@ TRACE_EVENT(i915_pipe_update_end, TP_fast_assign( __entry->pipe = crtc->pipe; __entry->frame = frame; - __entry->scanline = intel_get_crtc_scanline(crtc); + __entry->scanline = scanline_end; ), TP_printk("pipe %c, frame=%u, scanline=%u", @@ -107,6 +107,26 @@ TRACE_EVENT(i915_gem_object_create, TP_printk("obj=%p, size=%u", __entry->obj, __entry->size) ); +TRACE_EVENT(i915_gem_shrink, + TP_PROTO(struct drm_i915_private *i915, unsigned long target, unsigned flags), + TP_ARGS(i915, target, flags), + + TP_STRUCT__entry( + __field(int, dev) + __field(unsigned long, target) + __field(unsigned, flags) + ), + + TP_fast_assign( + __entry->dev = i915->dev->primary->index; + __entry->target = target; + __entry->flags = flags; + ), + + TP_printk("dev=%d, target=%lu, flags=%x", + __entry->dev, __entry->target, __entry->flags) +); + TRACE_EVENT(i915_vma_bind, TP_PROTO(struct i915_vma *vma, unsigned flags), TP_ARGS(vma, flags), @@ -186,33 +206,49 @@ DEFINE_EVENT(i915_va, i915_va_alloc, TP_ARGS(vm, start, length, name) ); -DECLARE_EVENT_CLASS(i915_page_table_entry, - TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), - TP_ARGS(vm, pde, start, pde_shift), +DECLARE_EVENT_CLASS(i915_px_entry, + TP_PROTO(struct i915_address_space *vm, u32 px, u64 start, u64 px_shift), + TP_ARGS(vm, px, start, px_shift), TP_STRUCT__entry( __field(struct i915_address_space *, vm) - __field(u32, pde) + __field(u32, px) __field(u64, start) __field(u64, end) ), TP_fast_assign( __entry->vm = vm; - __entry->pde = pde; + __entry->px = px; __entry->start = start; - __entry->end = ((start + (1ULL << pde_shift)) & ~((1ULL << pde_shift)-1)) - 1; + __entry->end = ((start + (1ULL << px_shift)) & ~((1ULL << px_shift)-1)) - 1; ), TP_printk("vm=%p, pde=%d (0x%llx-0x%llx)", - __entry->vm, __entry->pde, __entry->start, __entry->end) + __entry->vm, __entry->px, __entry->start, __entry->end) ); -DEFINE_EVENT(i915_page_table_entry, i915_page_table_entry_alloc, +DEFINE_EVENT(i915_px_entry, i915_page_table_entry_alloc, TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), TP_ARGS(vm, pde, start, pde_shift) ); +DEFINE_EVENT_PRINT(i915_px_entry, i915_page_directory_entry_alloc, + TP_PROTO(struct i915_address_space *vm, u32 pdpe, u64 start, u64 pdpe_shift), + TP_ARGS(vm, pdpe, start, pdpe_shift), + + TP_printk("vm=%p, pdpe=%d (0x%llx-0x%llx)", + __entry->vm, __entry->px, __entry->start, __entry->end) +); + +DEFINE_EVENT_PRINT(i915_px_entry, i915_page_directory_pointer_entry_alloc, + TP_PROTO(struct i915_address_space *vm, u32 pml4e, u64 start, u64 pml4e_shift), + TP_ARGS(vm, pml4e, start, pml4e_shift), + + TP_printk("vm=%p, pml4e=%d (0x%llx-0x%llx)", + __entry->vm, __entry->px, __entry->start, __entry->end) +); + /* Avoid extra math because we only support two sizes. The format is defined by * bitmap_scnprintf. Each 32 bits is 8 HEX digits followed by comma */ #define TRACE_PT_SIZE(bits) \ @@ -424,10 +460,10 @@ TRACE_EVENT(i915_gem_evict_vm, ); TRACE_EVENT(i915_gem_ring_sync_to, - TP_PROTO(struct intel_engine_cs *from, - struct intel_engine_cs *to, + TP_PROTO(struct drm_i915_gem_request *to_req, + struct intel_engine_cs *from, struct drm_i915_gem_request *req), - TP_ARGS(from, to, req), + TP_ARGS(to_req, from, req), TP_STRUCT__entry( __field(u32, dev) @@ -439,7 +475,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, TP_fast_assign( __entry->dev = from->dev->primary->index; __entry->sync_from = from->id; - __entry->sync_to = to->id; + __entry->sync_to = to_req->ring->id; __entry->seqno = i915_gem_request_get_seqno(req); ), @@ -475,8 +511,8 @@ TRACE_EVENT(i915_gem_ring_dispatch, ); TRACE_EVENT(i915_gem_ring_flush, - TP_PROTO(struct intel_engine_cs *ring, u32 invalidate, u32 flush), - TP_ARGS(ring, invalidate, flush), + TP_PROTO(struct drm_i915_gem_request *req, u32 invalidate, u32 flush), + TP_ARGS(req, invalidate, flush), TP_STRUCT__entry( __field(u32, dev) @@ -486,8 +522,8 @@ TRACE_EVENT(i915_gem_ring_flush, ), TP_fast_assign( - __entry->dev = ring->dev->primary->index; - __entry->ring = ring->id; + __entry->dev = req->ring->dev->primary->index; + __entry->ring = req->ring->id; __entry->invalidate = invalidate; __entry->flush = flush; ), @@ -628,7 +664,7 @@ TRACE_EVENT(i915_flip_complete, ); TRACE_EVENT_CONDITION(i915_reg_rw, - TP_PROTO(bool write, u32 reg, u64 val, int len, bool trace), + TP_PROTO(bool write, i915_reg_t reg, u64 val, int len, bool trace), TP_ARGS(write, reg, val, len, trace), @@ -643,7 +679,7 @@ TRACE_EVENT_CONDITION(i915_reg_rw, TP_fast_assign( __entry->val = (u64)val; - __entry->reg = reg; + __entry->reg = i915_mmio_reg_offset(reg); __entry->write = write; __entry->len = len; ), diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c index 5eee75bff170..dea7429be4d0 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.c +++ b/drivers/gpu/drm/i915/i915_vgpu.c @@ -69,13 +69,13 @@ void i915_check_vgpu(struct drm_device *dev) if (!IS_HASWELL(dev)) return; - magic = readq(dev_priv->regs + vgtif_reg(magic)); + magic = __raw_i915_read64(dev_priv, vgtif_reg(magic)); if (magic != VGT_MAGIC) return; version = INTEL_VGT_IF_VERSION_ENCODE( - readw(dev_priv->regs + vgtif_reg(version_major)), - readw(dev_priv->regs + vgtif_reg(version_minor))); + __raw_i915_read16(dev_priv, vgtif_reg(version_major)), + __raw_i915_read16(dev_priv, vgtif_reg(version_minor))); if (version != INTEL_VGT_IF_VERSION) { DRM_INFO("VGT interface version mismatch!\n"); return; diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h index 97a88b5f6a26..3c83b47b5f69 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.h +++ b/drivers/gpu/drm/i915/i915_vgpu.h @@ -40,6 +40,19 @@ #define INTEL_VGT_IF_VERSION \ INTEL_VGT_IF_VERSION_ENCODE(VGT_VERSION_MAJOR, VGT_VERSION_MINOR) +/* + * notifications from guest to vgpu device model + */ +enum vgt_g2v_type { + VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2, + VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY, + VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE, + VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY, + VGT_G2V_EXECLIST_CONTEXT_CREATE, + VGT_G2V_EXECLIST_CONTEXT_DESTROY, + VGT_G2V_MAX, +}; + struct vgt_if { uint64_t magic; /* VGT_MAGIC */ uint16_t version_major; @@ -70,15 +83,28 @@ struct vgt_if { uint32_t rsv3[0x200 - 24]; /* pad to half page */ /* * The bottom half page is for response from Gfx driver to hypervisor. - * Set to reserved fields temporarily by now. */ uint32_t rsv4; uint32_t display_ready; /* ready for display owner switch */ - uint32_t rsv5[0x200 - 2]; /* pad to one page */ + + uint32_t rsv5[4]; + + uint32_t g2v_notify; + uint32_t rsv6[7]; + + struct { + uint32_t lo; + uint32_t hi; + } pdp[4]; + + uint32_t execlist_context_descriptor_lo; + uint32_t execlist_context_descriptor_hi; + + uint32_t rsv7[0x200 - 24]; /* pad to one page */ } __packed; #define vgtif_reg(x) \ - (VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x) + _MMIO((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x)) /* vGPU display status to be used by the host side */ #define VGT_DRV_DISPLAY_NOT_READY 0 diff --git a/drivers/gpu/drm/i915/intel_acpi.c b/drivers/gpu/drm/i915/intel_acpi.c index d96eee1ae9c5..eb638a1e69d2 100644 --- a/drivers/gpu/drm/i915/intel_acpi.c +++ b/drivers/gpu/drm/i915/intel_acpi.c @@ -5,7 +5,6 @@ */ #include <linux/pci.h> #include <linux/acpi.h> -#include <linux/vga_switcheroo.h> #include <drm/drmP.h> #include "i915_drv.h" @@ -146,7 +145,7 @@ static bool intel_dsm_detect(void) if (vga_count == 2 && has_dsm) { acpi_get_name(intel_dsm_priv.dhandle, ACPI_FULL_PATHNAME, &buffer); - DRM_DEBUG_DRIVER("VGA switcheroo: detected DSM switching method %s handle\n", + DRM_DEBUG_DRIVER("vga_switcheroo: detected DSM switching method %s handle\n", acpi_method_name); return true; } diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c index 8e35e0d013df..643f342de33b 100644 --- a/drivers/gpu/drm/i915/intel_atomic.c +++ b/drivers/gpu/drm/i915/intel_atomic.c @@ -35,133 +35,6 @@ #include <drm/drm_plane_helper.h> #include "intel_drv.h" - -/** - * intel_atomic_check - validate state object - * @dev: drm device - * @state: state to validate - */ -int intel_atomic_check(struct drm_device *dev, - struct drm_atomic_state *state) -{ - int nplanes = dev->mode_config.num_total_plane; - int ncrtcs = dev->mode_config.num_crtc; - int nconnectors = dev->mode_config.num_connector; - enum pipe nuclear_pipe = INVALID_PIPE; - struct intel_crtc *nuclear_crtc = NULL; - struct intel_crtc_state *crtc_state = NULL; - int ret; - int i; - bool not_nuclear = false; - - /* - * FIXME: At the moment, we only support "nuclear pageflip" on a - * single CRTC. Cross-crtc updates will be added later. - */ - for (i = 0; i < nplanes; i++) { - struct intel_plane *plane = to_intel_plane(state->planes[i]); - if (!plane) - continue; - - if (nuclear_pipe == INVALID_PIPE) { - nuclear_pipe = plane->pipe; - } else if (nuclear_pipe != plane->pipe) { - DRM_DEBUG_KMS("i915 only support atomic plane operations on a single CRTC at the moment\n"); - return -EINVAL; - } - } - - /* - * FIXME: We only handle planes for now; make sure there are no CRTC's - * or connectors involved. - */ - state->allow_modeset = false; - for (i = 0; i < ncrtcs; i++) { - struct intel_crtc *crtc = to_intel_crtc(state->crtcs[i]); - if (crtc) - memset(&crtc->atomic, 0, sizeof(crtc->atomic)); - if (crtc && crtc->pipe != nuclear_pipe) - not_nuclear = true; - if (crtc && crtc->pipe == nuclear_pipe) { - nuclear_crtc = crtc; - crtc_state = to_intel_crtc_state(state->crtc_states[i]); - } - } - for (i = 0; i < nconnectors; i++) - if (state->connectors[i] != NULL) - not_nuclear = true; - - if (not_nuclear) { - DRM_DEBUG_KMS("i915 only supports atomic plane operations at the moment\n"); - return -EINVAL; - } - - ret = drm_atomic_helper_check_planes(dev, state); - if (ret) - return ret; - - /* FIXME: move to crtc atomic check function once it is ready */ - ret = intel_atomic_setup_scalers(dev, nuclear_crtc, crtc_state); - if (ret) - return ret; - - return ret; -} - - -/** - * intel_atomic_commit - commit validated state object - * @dev: DRM device - * @state: the top-level driver state object - * @async: asynchronous commit - * - * This function commits a top-level state object that has been validated - * with drm_atomic_helper_check(). - * - * FIXME: Atomic modeset support for i915 is not yet complete. At the moment - * we can only handle plane-related operations and do not yet support - * asynchronous commit. - * - * RETURNS - * Zero for success or -errno. - */ -int intel_atomic_commit(struct drm_device *dev, - struct drm_atomic_state *state, - bool async) -{ - struct drm_crtc_state *crtc_state; - struct drm_crtc *crtc; - int ret, i; - - if (async) { - DRM_DEBUG_KMS("i915 does not yet support async commit\n"); - return -EINVAL; - } - - ret = drm_atomic_helper_prepare_planes(dev, state); - if (ret) - return ret; - - /* Point of no return */ - drm_atomic_helper_swap_state(dev, state); - - /* swap crtc_scaler_state */ - for_each_crtc_in_state(state, crtc, crtc_state, i) { - to_intel_crtc(crtc)->config = to_intel_crtc_state(crtc->state); - - if (INTEL_INFO(dev)->gen >= 9) - skl_detach_scalers(to_intel_crtc(crtc)); - - drm_atomic_helper_commit_planes_on_crtc(crtc_state); - } - - drm_atomic_helper_wait_for_vblanks(dev, state); - drm_atomic_helper_cleanup_planes(dev, state); - drm_atomic_state_free(state); - - return 0; -} - /** * intel_connector_atomic_get_property - fetch connector property value * @connector: connector to fetch property for @@ -212,21 +85,16 @@ intel_connector_atomic_get_property(struct drm_connector *connector, struct drm_crtc_state * intel_crtc_duplicate_state(struct drm_crtc *crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *crtc_state; - if (WARN_ON(!intel_crtc->config)) - crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL); - else - crtc_state = kmemdup(intel_crtc->config, - sizeof(*intel_crtc->config), GFP_KERNEL); - + crtc_state = kmemdup(crtc->state, sizeof(*crtc_state), GFP_KERNEL); if (!crtc_state) return NULL; __drm_atomic_helper_crtc_duplicate_state(crtc, &crtc_state->base); - crtc_state->base.crtc = crtc; + crtc_state->update_pipe = false; + crtc_state->disable_lp_wm = false; return &crtc_state->base; } @@ -269,21 +137,13 @@ int intel_atomic_setup_scalers(struct drm_device *dev, struct drm_plane *plane = NULL; struct intel_plane *intel_plane; struct intel_plane_state *plane_state = NULL; - struct intel_crtc_scaler_state *scaler_state; - struct drm_atomic_state *drm_state; + struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; + struct drm_atomic_state *drm_state = crtc_state->base.state; int num_scalers_need; int i, j; - if (INTEL_INFO(dev)->gen < 9 || !intel_crtc || !crtc_state) - return 0; - - scaler_state = &crtc_state->scaler_state; - drm_state = crtc_state->base.state; - num_scalers_need = hweight32(scaler_state->scaler_users); - DRM_DEBUG_KMS("crtc_state = %p need = %d avail = %d scaler_users = 0x%x\n", - crtc_state, num_scalers_need, intel_crtc->num_scalers, - scaler_state->scaler_users); /* * High level flow: @@ -307,17 +167,21 @@ int intel_atomic_setup_scalers(struct drm_device *dev, /* walkthrough scaler_users bits and start assigning scalers */ for (i = 0; i < sizeof(scaler_state->scaler_users) * 8; i++) { int *scaler_id; + const char *name; + int idx; /* skip if scaler not required */ if (!(scaler_state->scaler_users & (1 << i))) continue; if (i == SKL_CRTC_INDEX) { + name = "CRTC"; + idx = intel_crtc->base.base.id; + /* panel fitter case: assign as a crtc scaler */ scaler_id = &scaler_state->scaler_id; } else { - if (!drm_state) - continue; + name = "PLANE"; /* plane scaler case: assign as a plane scaler */ /* find the plane that set the bit as scaler_user */ @@ -336,9 +200,17 @@ int intel_atomic_setup_scalers(struct drm_device *dev, plane->base.id); return PTR_ERR(state); } + + /* + * the plane is added after plane checks are run, + * but since this plane is unchanged just do the + * minimum required validation. + */ + crtc_state->base.planes_changed = true; } intel_plane = to_intel_plane(plane); + idx = plane->base.id; /* plane on different crtc cannot be a scaler user of this crtc */ if (WARN_ON(intel_plane->pipe != intel_crtc->pipe)) { @@ -354,23 +226,16 @@ int intel_atomic_setup_scalers(struct drm_device *dev, for (j = 0; j < intel_crtc->num_scalers; j++) { if (!scaler_state->scalers[j].in_use) { scaler_state->scalers[j].in_use = 1; - *scaler_id = scaler_state->scalers[j].id; + *scaler_id = j; DRM_DEBUG_KMS("Attached scaler id %u.%u to %s:%d\n", - intel_crtc->pipe, - i == SKL_CRTC_INDEX ? scaler_state->scaler_id : - plane_state->scaler_id, - i == SKL_CRTC_INDEX ? "CRTC" : "PLANE", - i == SKL_CRTC_INDEX ? intel_crtc->base.base.id : - plane->base.id); + intel_crtc->pipe, *scaler_id, name, idx); break; } } } if (WARN_ON(*scaler_id < 0)) { - DRM_DEBUG_KMS("Cannot find scaler for %s:%d\n", - i == SKL_CRTC_INDEX ? "CRTC" : "PLANE", - i == SKL_CRTC_INDEX ? intel_crtc->base.base.id:plane->base.id); + DRM_DEBUG_KMS("Cannot find scaler for %s:%d\n", name, idx); continue; } @@ -392,3 +257,54 @@ int intel_atomic_setup_scalers(struct drm_device *dev, return 0; } + +static void +intel_atomic_duplicate_dpll_state(struct drm_i915_private *dev_priv, + struct intel_shared_dpll_config *shared_dpll) +{ + enum intel_dpll_id i; + + /* Copy shared dpll state */ + for (i = 0; i < dev_priv->num_shared_dpll; i++) { + struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; + + shared_dpll[i] = pll->config; + } +} + +struct intel_shared_dpll_config * +intel_atomic_get_shared_dpll_state(struct drm_atomic_state *s) +{ + struct intel_atomic_state *state = to_intel_atomic_state(s); + + WARN_ON(!drm_modeset_is_locked(&s->dev->mode_config.connection_mutex)); + + if (!state->dpll_set) { + state->dpll_set = true; + + intel_atomic_duplicate_dpll_state(to_i915(s->dev), + state->shared_dpll); + } + + return state->shared_dpll; +} + +struct drm_atomic_state * +intel_atomic_state_alloc(struct drm_device *dev) +{ + struct intel_atomic_state *state = kzalloc(sizeof(*state), GFP_KERNEL); + + if (!state || drm_atomic_state_init(dev, &state->base) < 0) { + kfree(state); + return NULL; + } + + return &state->base; +} + +void intel_atomic_state_clear(struct drm_atomic_state *s) +{ + struct intel_atomic_state *state = to_intel_atomic_state(s); + drm_atomic_state_default_clear(&state->base); + state->dpll_set = false; +} diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c index 86ba4b2c3a65..c6bb0fc1edfb 100644 --- a/drivers/gpu/drm/i915/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/intel_atomic_plane.c @@ -56,6 +56,7 @@ intel_create_plane_state(struct drm_plane *plane) state->base.plane = plane; state->base.rotation = BIT(DRM_ROTATE_0); + state->ckey.flags = I915_SET_COLORKEY_NONE; return state; } @@ -75,11 +76,7 @@ intel_plane_duplicate_state(struct drm_plane *plane) struct drm_plane_state *state; struct intel_plane_state *intel_state; - if (WARN_ON(!plane->state)) - intel_state = intel_create_plane_state(plane); - else - intel_state = kmemdup(plane->state, sizeof(*intel_state), - GFP_KERNEL); + intel_state = kmemdup(plane->state, sizeof(*intel_state), GFP_KERNEL); if (!intel_state) return NULL; @@ -87,6 +84,7 @@ intel_plane_duplicate_state(struct drm_plane *plane) state = &intel_state->base; __drm_atomic_helper_plane_duplicate_state(plane, state); + intel_state->wait_req = NULL; return state; } @@ -103,6 +101,7 @@ void intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state) { + WARN_ON(state && to_intel_plane_state(state)->wait_req); drm_atomic_helper_plane_destroy_state(plane, state); } @@ -114,8 +113,10 @@ static int intel_plane_atomic_check(struct drm_plane *plane, struct intel_crtc_state *crtc_state; struct intel_plane *intel_plane = to_intel_plane(plane); struct intel_plane_state *intel_state = to_intel_plane_state(state); + struct drm_crtc_state *drm_crtc_state; + int ret; - crtc = crtc ? crtc : plane->crtc; + crtc = crtc ? crtc : plane->state->crtc; intel_crtc = to_intel_crtc(crtc); /* @@ -127,16 +128,11 @@ static int intel_plane_atomic_check(struct drm_plane *plane, if (!crtc) return 0; - /* FIXME: temporary hack necessary while we still use the plane update - * helper. */ - if (state->state) { - crtc_state = - intel_atomic_get_crtc_state(state->state, intel_crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); - } else { - crtc_state = intel_crtc->config; - } + drm_crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc); + if (WARN_ON(!drm_crtc_state)) + return -EINVAL; + + crtc_state = to_intel_crtc_state(drm_crtc_state); /* * The original src/dest coordinates are stored in state->base, but @@ -160,20 +156,6 @@ static int intel_plane_atomic_check(struct drm_plane *plane, intel_state->clip.y2 = crtc_state->base.active ? crtc_state->pipe_src_h : 0; - /* - * Disabling a plane is always okay; we just need to update - * fb tracking in a special way since cleanup_fb() won't - * get called by the plane helpers. - */ - if (state->fb == NULL && plane->state->fb != NULL) { - /* - * 'prepare' is never called when plane is being disabled, so - * we need to handle frontbuffer tracking as a special case - */ - intel_crtc->atomic.disabled_planes |= - (1 << drm_plane_index(plane)); - } - if (state->fb && intel_rotation_90_or_270(state->rotation)) { if (!(state->fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || state->fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)) { @@ -198,7 +180,12 @@ static int intel_plane_atomic_check(struct drm_plane *plane, } } - return intel_plane->check_plane(plane, intel_state); + intel_state->visible = false; + ret = intel_plane->check_plane(plane, crtc_state, intel_state); + if (ret) + return ret; + + return intel_plane_atomic_calc_changes(&crtc_state->base, state); } static void intel_plane_atomic_update(struct drm_plane *plane, diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c index 3da9b8409f20..9aa83e71b792 100644 --- a/drivers/gpu/drm/i915/intel_audio.c +++ b/drivers/gpu/drm/i915/intel_audio.c @@ -41,7 +41,8 @@ * * The disable sequences must be performed before disabling the transcoder or * port. The enable sequences may only be performed after enabling the - * transcoder and port, and after completed link training. + * transcoder and port, and after completed link training. Therefore the audio + * enable/disable sequences are part of the modeset sequence. * * The codec and controller sequences could be done either parallel or serial, * but generally the ELDV/PD change in the codec sequence indicates to the audio @@ -49,36 +50,67 @@ * co-operation between the graphics and audio drivers is handled via audio * related registers. (The notable exception is the power management, not * covered here.) + * + * The struct i915_audio_component is used to interact between the graphics + * and audio drivers. The struct i915_audio_component_ops *ops in it is + * defined in graphics driver and called in audio driver. The + * struct i915_audio_component_audio_ops *audio_ops is called from i915 driver. */ static const struct { int clock; u32 config; } hdmi_audio_clock[] = { - { DIV_ROUND_UP(25200 * 1000, 1001), AUD_CONFIG_PIXEL_CLOCK_HDMI_25175 }, + { 25175, AUD_CONFIG_PIXEL_CLOCK_HDMI_25175 }, { 25200, AUD_CONFIG_PIXEL_CLOCK_HDMI_25200 }, /* default per bspec */ { 27000, AUD_CONFIG_PIXEL_CLOCK_HDMI_27000 }, - { 27000 * 1001 / 1000, AUD_CONFIG_PIXEL_CLOCK_HDMI_27027 }, + { 27027, AUD_CONFIG_PIXEL_CLOCK_HDMI_27027 }, { 54000, AUD_CONFIG_PIXEL_CLOCK_HDMI_54000 }, - { 54000 * 1001 / 1000, AUD_CONFIG_PIXEL_CLOCK_HDMI_54054 }, - { DIV_ROUND_UP(74250 * 1000, 1001), AUD_CONFIG_PIXEL_CLOCK_HDMI_74176 }, + { 54054, AUD_CONFIG_PIXEL_CLOCK_HDMI_54054 }, + { 74176, AUD_CONFIG_PIXEL_CLOCK_HDMI_74176 }, { 74250, AUD_CONFIG_PIXEL_CLOCK_HDMI_74250 }, - { DIV_ROUND_UP(148500 * 1000, 1001), AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 }, + { 148352, AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 }, { 148500, AUD_CONFIG_PIXEL_CLOCK_HDMI_148500 }, }; +/* HDMI N/CTS table */ +#define TMDS_297M 297000 +#define TMDS_296M 296703 +static const struct { + int sample_rate; + int clock; + int n; + int cts; +} aud_ncts[] = { + { 44100, TMDS_296M, 4459, 234375 }, + { 44100, TMDS_297M, 4704, 247500 }, + { 48000, TMDS_296M, 5824, 281250 }, + { 48000, TMDS_297M, 5120, 247500 }, + { 32000, TMDS_296M, 5824, 421875 }, + { 32000, TMDS_297M, 3072, 222750 }, + { 88200, TMDS_296M, 8918, 234375 }, + { 88200, TMDS_297M, 9408, 247500 }, + { 96000, TMDS_296M, 11648, 281250 }, + { 96000, TMDS_297M, 10240, 247500 }, + { 176400, TMDS_296M, 17836, 234375 }, + { 176400, TMDS_297M, 18816, 247500 }, + { 192000, TMDS_296M, 23296, 281250 }, + { 192000, TMDS_297M, 20480, 247500 }, +}; + /* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */ -static u32 audio_config_hdmi_pixel_clock(struct drm_display_mode *mode) +static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted_mode) { int i; for (i = 0; i < ARRAY_SIZE(hdmi_audio_clock); i++) { - if (mode->clock == hdmi_audio_clock[i].clock) + if (adjusted_mode->crtc_clock == hdmi_audio_clock[i].clock) break; } if (i == ARRAY_SIZE(hdmi_audio_clock)) { - DRM_DEBUG_KMS("HDMI audio pixel clock setting for %d not found, falling back to defaults\n", mode->clock); + DRM_DEBUG_KMS("HDMI audio pixel clock setting for %d not found, falling back to defaults\n", + adjusted_mode->crtc_clock); i = 1; } @@ -89,10 +121,49 @@ static u32 audio_config_hdmi_pixel_clock(struct drm_display_mode *mode) return hdmi_audio_clock[i].config; } +static int audio_config_get_n(const struct drm_display_mode *mode, int rate) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aud_ncts); i++) { + if ((rate == aud_ncts[i].sample_rate) && + (mode->clock == aud_ncts[i].clock)) { + return aud_ncts[i].n; + } + } + return 0; +} + +static uint32_t audio_config_setup_n_reg(int n, uint32_t val) +{ + int n_low, n_up; + uint32_t tmp = val; + + n_low = n & 0xfff; + n_up = (n >> 12) & 0xff; + tmp &= ~(AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK); + tmp |= ((n_up << AUD_CONFIG_UPPER_N_SHIFT) | + (n_low << AUD_CONFIG_LOWER_N_SHIFT) | + AUD_CONFIG_N_PROG_ENABLE); + return tmp; +} + +/* check whether N/CTS/M need be set manually */ +static bool audio_rate_need_prog(struct intel_crtc *crtc, + const struct drm_display_mode *mode) +{ + if (((mode->clock == TMDS_297M) || + (mode->clock == TMDS_296M)) && + intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) + return true; + else + return false; +} + static bool intel_eld_uptodate(struct drm_connector *connector, - int reg_eldv, uint32_t bits_eldv, - int reg_elda, uint32_t bits_elda, - int reg_edid) + i915_reg_t reg_eldv, uint32_t bits_eldv, + i915_reg_t reg_elda, uint32_t bits_elda, + i915_reg_t reg_edid) { struct drm_i915_private *dev_priv = connector->dev->dev_private; uint8_t *eld = connector->eld; @@ -137,7 +208,7 @@ static void g4x_audio_codec_disable(struct intel_encoder *encoder) static void g4x_audio_codec_enable(struct drm_connector *connector, struct intel_encoder *encoder, - struct drm_display_mode *mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = connector->dev->dev_private; uint8_t *eld = connector->eld; @@ -183,6 +254,8 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder) DRM_DEBUG_KMS("Disable audio codec on pipe %c\n", pipe_name(pipe)); + mutex_lock(&dev_priv->av_mutex); + /* Disable timestamps */ tmp = I915_READ(HSW_AUD_CFG(pipe)); tmp &= ~AUD_CONFIG_N_VALUE_INDEX; @@ -198,22 +271,31 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder) tmp &= ~AUDIO_ELD_VALID(pipe); tmp &= ~AUDIO_OUTPUT_ENABLE(pipe); I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp); + + mutex_unlock(&dev_priv->av_mutex); } static void hsw_audio_codec_enable(struct drm_connector *connector, struct intel_encoder *encoder, - struct drm_display_mode *mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = connector->dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); enum pipe pipe = intel_crtc->pipe; + struct i915_audio_component *acomp = dev_priv->audio_component; const uint8_t *eld = connector->eld; + struct intel_digital_port *intel_dig_port = + enc_to_dig_port(&encoder->base); + enum port port = intel_dig_port->port; uint32_t tmp; int len, i; + int n, rate; DRM_DEBUG_KMS("Enable audio codec on pipe %c, %u bytes ELD\n", pipe_name(pipe), drm_eld_size(eld)); + mutex_lock(&dev_priv->av_mutex); + /* Enable audio presence detect, invalidate ELD */ tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); tmp |= AUDIO_OUTPUT_ENABLE(pipe); @@ -245,13 +327,32 @@ static void hsw_audio_codec_enable(struct drm_connector *connector, /* Enable timestamps */ tmp = I915_READ(HSW_AUD_CFG(pipe)); tmp &= ~AUD_CONFIG_N_VALUE_INDEX; - tmp &= ~AUD_CONFIG_N_PROG_ENABLE; tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT)) tmp |= AUD_CONFIG_N_VALUE_INDEX; else - tmp |= audio_config_hdmi_pixel_clock(mode); + tmp |= audio_config_hdmi_pixel_clock(adjusted_mode); + + tmp &= ~AUD_CONFIG_N_PROG_ENABLE; + if (audio_rate_need_prog(intel_crtc, adjusted_mode)) { + if (!acomp) + rate = 0; + else if (port >= PORT_A && port <= PORT_E) + rate = acomp->aud_sample_rate[port]; + else { + DRM_ERROR("invalid port: %d\n", port); + rate = 0; + } + n = audio_config_get_n(adjusted_mode, rate); + if (n != 0) + tmp = audio_config_setup_n_reg(n, tmp); + else + DRM_DEBUG_KMS("no suitable N value is found\n"); + } + I915_WRITE(HSW_AUD_CFG(pipe), tmp); + + mutex_unlock(&dev_priv->av_mutex); } static void ilk_audio_codec_disable(struct intel_encoder *encoder) @@ -263,8 +364,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder) enum port port = intel_dig_port->port; enum pipe pipe = intel_crtc->pipe; uint32_t tmp, eldv; - int aud_config; - int aud_cntrl_st2; + i915_reg_t aud_config, aud_cntrl_st2; DRM_DEBUG_KMS("Disable audio codec on port %c, pipe %c\n", port_name(port), pipe_name(pipe)); @@ -303,7 +403,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder) static void ilk_audio_codec_enable(struct drm_connector *connector, struct intel_encoder *encoder, - struct drm_display_mode *mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = connector->dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); @@ -315,10 +415,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector, uint32_t eldv; uint32_t tmp; int len, i; - int hdmiw_hdmiedid; - int aud_config; - int aud_cntl_st; - int aud_cntrl_st2; + i915_reg_t hdmiw_hdmiedid, aud_config, aud_cntl_st, aud_cntrl_st2; DRM_DEBUG_KMS("Enable audio codec on port %c, pipe %c, %u bytes ELD\n", port_name(port), pipe_name(pipe), drm_eld_size(eld)); @@ -380,7 +477,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector, if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT)) tmp |= AUD_CONFIG_N_VALUE_INDEX; else - tmp |= audio_config_hdmi_pixel_clock(mode); + tmp |= audio_config_hdmi_pixel_clock(adjusted_mode); I915_WRITE(aud_config, tmp); } @@ -395,12 +492,15 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder) { struct drm_encoder *encoder = &intel_encoder->base; struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); - struct drm_display_mode *mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; struct drm_connector *connector; struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_audio_component *acomp = dev_priv->audio_component; + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + enum port port = intel_dig_port->port; - connector = drm_select_eld(encoder, mode); + connector = drm_select_eld(encoder); if (!connector) return; @@ -415,26 +515,37 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder) if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) connector->eld[5] |= (1 << 2); - connector->eld[6] = drm_av_sync_delay(connector, mode) / 2; + connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; if (dev_priv->display.audio_codec_enable) - dev_priv->display.audio_codec_enable(connector, intel_encoder, mode); + dev_priv->display.audio_codec_enable(connector, intel_encoder, + adjusted_mode); + + if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) + acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port); } /** * intel_audio_codec_disable - Disable the audio codec for HD audio - * @encoder: encoder on which to disable audio + * @intel_encoder: encoder on which to disable audio * * The disable sequences must be performed before disabling the transcoder or * port. */ -void intel_audio_codec_disable(struct intel_encoder *encoder) +void intel_audio_codec_disable(struct intel_encoder *intel_encoder) { - struct drm_device *dev = encoder->base.dev; + struct drm_encoder *encoder = &intel_encoder->base; + struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_audio_component *acomp = dev_priv->audio_component; + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + enum port port = intel_dig_port->port; if (dev_priv->display.audio_codec_disable) - dev_priv->display.audio_codec_disable(encoder); + dev_priv->display.audio_codec_disable(intel_encoder); + + if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) + acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port); } /** @@ -476,7 +587,7 @@ static void i915_audio_component_codec_wake_override(struct device *dev, struct drm_i915_private *dev_priv = dev_to_i915(dev); u32 tmp; - if (!IS_SKYLAKE(dev_priv)) + if (!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv)) return; /* @@ -513,24 +624,112 @@ static int i915_audio_component_get_cdclk_freq(struct device *dev) return ret; } +static int i915_audio_component_sync_audio_rate(struct device *dev, + int port, int rate) +{ + struct drm_i915_private *dev_priv = dev_to_i915(dev); + struct drm_device *drm_dev = dev_priv->dev; + struct intel_encoder *intel_encoder; + struct intel_digital_port *intel_dig_port; + struct intel_crtc *crtc; + struct drm_display_mode *mode; + struct i915_audio_component *acomp = dev_priv->audio_component; + enum pipe pipe = -1; + u32 tmp; + int n; + + /* HSW, BDW, SKL, KBL need this fix */ + if (!IS_SKYLAKE(dev_priv) && + !IS_KABYLAKE(dev_priv) && + !IS_BROADWELL(dev_priv) && + !IS_HASWELL(dev_priv)) + return 0; + + mutex_lock(&dev_priv->av_mutex); + /* 1. get the pipe */ + for_each_intel_encoder(drm_dev, intel_encoder) { + if (intel_encoder->type != INTEL_OUTPUT_HDMI) + continue; + intel_dig_port = enc_to_dig_port(&intel_encoder->base); + if (port == intel_dig_port->port) { + crtc = to_intel_crtc(intel_encoder->base.crtc); + if (!crtc) { + DRM_DEBUG_KMS("%s: crtc is NULL\n", __func__); + continue; + } + pipe = crtc->pipe; + break; + } + } + + if (pipe == INVALID_PIPE) { + DRM_DEBUG_KMS("no pipe for the port %c\n", port_name(port)); + mutex_unlock(&dev_priv->av_mutex); + return -ENODEV; + } + DRM_DEBUG_KMS("pipe %c connects port %c\n", + pipe_name(pipe), port_name(port)); + mode = &crtc->config->base.adjusted_mode; + + /* port must be valid now, otherwise the pipe will be invalid */ + acomp->aud_sample_rate[port] = rate; + + /* 2. check whether to set the N/CTS/M manually or not */ + if (!audio_rate_need_prog(crtc, mode)) { + tmp = I915_READ(HSW_AUD_CFG(pipe)); + tmp &= ~AUD_CONFIG_N_PROG_ENABLE; + I915_WRITE(HSW_AUD_CFG(pipe), tmp); + mutex_unlock(&dev_priv->av_mutex); + return 0; + } + + n = audio_config_get_n(mode, rate); + if (n == 0) { + DRM_DEBUG_KMS("Using automatic mode for N value on port %c\n", + port_name(port)); + tmp = I915_READ(HSW_AUD_CFG(pipe)); + tmp &= ~AUD_CONFIG_N_PROG_ENABLE; + I915_WRITE(HSW_AUD_CFG(pipe), tmp); + mutex_unlock(&dev_priv->av_mutex); + return 0; + } + + /* 3. set the N/CTS/M */ + tmp = I915_READ(HSW_AUD_CFG(pipe)); + tmp = audio_config_setup_n_reg(n, tmp); + I915_WRITE(HSW_AUD_CFG(pipe), tmp); + + mutex_unlock(&dev_priv->av_mutex); + return 0; +} + static const struct i915_audio_component_ops i915_audio_component_ops = { .owner = THIS_MODULE, .get_power = i915_audio_component_get_power, .put_power = i915_audio_component_put_power, .codec_wake_override = i915_audio_component_codec_wake_override, .get_cdclk_freq = i915_audio_component_get_cdclk_freq, + .sync_audio_rate = i915_audio_component_sync_audio_rate, }; static int i915_audio_component_bind(struct device *i915_dev, struct device *hda_dev, void *data) { struct i915_audio_component *acomp = data; + struct drm_i915_private *dev_priv = dev_to_i915(i915_dev); + int i; if (WARN_ON(acomp->ops || acomp->dev)) return -EEXIST; + drm_modeset_lock_all(dev_priv->dev); acomp->ops = &i915_audio_component_ops; acomp->dev = i915_dev; + BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS); + for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++) + acomp->aud_sample_rate[i] = 0; + dev_priv->audio_component = acomp; + drm_modeset_unlock_all(dev_priv->dev); return 0; } @@ -539,9 +738,13 @@ static void i915_audio_component_unbind(struct device *i915_dev, struct device *hda_dev, void *data) { struct i915_audio_component *acomp = data; + struct drm_i915_private *dev_priv = dev_to_i915(i915_dev); + drm_modeset_lock_all(dev_priv->dev); acomp->ops = NULL; acomp->dev = NULL; + dev_priv->audio_component = NULL; + drm_modeset_unlock_all(dev_priv->dev); } static const struct component_ops i915_audio_component_bind_ops = { diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index 198fc3c3291b..070470fe9a91 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -42,7 +42,7 @@ find_section(const void *_bdb, int section_id) const struct bdb_header *bdb = _bdb; const u8 *base = _bdb; int index = 0; - u16 total, current_size; + u32 total, current_size; u8 current_id; /* skip to first section */ @@ -57,6 +57,10 @@ find_section(const void *_bdb, int section_id) current_size = *((const u16 *)(base + index)); index += 2; + /* The MIPI Sequence Block v3+ has a separate size field. */ + if (current_id == BDB_MIPI_SEQUENCE && *(base + index) >= 3) + current_size = *((const u32 *)(base + index + 1)); + if (index + current_size > total) return NULL; @@ -122,42 +126,6 @@ fill_detail_timing_data(struct drm_display_mode *panel_fixed_mode, drm_mode_set_name(panel_fixed_mode); } -static bool -lvds_dvo_timing_equal_size(const struct lvds_dvo_timing *a, - const struct lvds_dvo_timing *b) -{ - if (a->hactive_hi != b->hactive_hi || - a->hactive_lo != b->hactive_lo) - return false; - - if (a->hsync_off_hi != b->hsync_off_hi || - a->hsync_off_lo != b->hsync_off_lo) - return false; - - if (a->hsync_pulse_width != b->hsync_pulse_width) - return false; - - if (a->hblank_hi != b->hblank_hi || - a->hblank_lo != b->hblank_lo) - return false; - - if (a->vactive_hi != b->vactive_hi || - a->vactive_lo != b->vactive_lo) - return false; - - if (a->vsync_off != b->vsync_off) - return false; - - if (a->vsync_pulse_width != b->vsync_pulse_width) - return false; - - if (a->vblank_hi != b->vblank_hi || - a->vblank_lo != b->vblank_lo) - return false; - - return true; -} - static const struct lvds_dvo_timing * get_lvds_dvo_timing(const struct bdb_lvds_lfp_data *lvds_lfp_data, const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs, @@ -213,7 +181,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv, const struct lvds_dvo_timing *panel_dvo_timing; const struct lvds_fp_timing *fp_timing; struct drm_display_mode *panel_fixed_mode; - int i, downclock, drrs_mode; + int drrs_mode; lvds_options = find_section(bdb, BDB_LVDS_OPTIONS); if (!lvds_options) @@ -272,30 +240,6 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv, DRM_DEBUG_KMS("Found panel mode in BIOS VBT tables:\n"); drm_mode_debug_printmodeline(panel_fixed_mode); - /* - * Iterate over the LVDS panel timing info to find the lowest clock - * for the native resolution. - */ - downclock = panel_dvo_timing->clock; - for (i = 0; i < 16; i++) { - const struct lvds_dvo_timing *dvo_timing; - - dvo_timing = get_lvds_dvo_timing(lvds_lfp_data, - lvds_lfp_data_ptrs, - i); - if (lvds_dvo_timing_equal_size(dvo_timing, panel_dvo_timing) && - dvo_timing->clock < downclock) - downclock = dvo_timing->clock; - } - - if (downclock < panel_dvo_timing->clock && i915.lvds_downclock) { - dev_priv->lvds_downclock_avail = 1; - dev_priv->lvds_downclock = downclock * 10; - DRM_DEBUG_KMS("LVDS downclock is found in VBT. " - "Normal Clock %dKHz, downclock %dKHz\n", - panel_fixed_mode->clock, 10*downclock); - } - fp_timing = get_lvds_fp_timing(bdb, lvds_lfp_data, lvds_lfp_data_ptrs, lvds_options->panel_type); @@ -412,7 +356,10 @@ parse_general_features(struct drm_i915_private *dev_priv, general = find_section(bdb, BDB_GENERAL_FEATURES); if (general) { dev_priv->vbt.int_tv_support = general->int_tv_support; - dev_priv->vbt.int_crt_support = general->int_crt_support; + /* int_crt_support can't be trusted on earlier platforms */ + if (bdb->version >= 155 && + (HAS_DDI(dev_priv) || IS_VALLEYVIEW(dev_priv))) + dev_priv->vbt.int_crt_support = general->int_crt_support; dev_priv->vbt.lvds_use_ssc = general->enable_ssc; dev_priv->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(dev, general->ssc_freq); @@ -461,7 +408,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv, { struct sdvo_device_mapping *p_mapping; const struct bdb_general_definitions *p_defs; - const union child_device_config *p_child; + const struct old_child_dev_config *child; /* legacy */ int i, child_device_num, count; u16 block_size; @@ -470,14 +417,14 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv, DRM_DEBUG_KMS("No general definition block is found, unable to construct sdvo mapping.\n"); return; } - /* judge whether the size of child device meets the requirements. - * If the child device size obtained from general definition block - * is different with sizeof(struct child_device_config), skip the - * parsing of sdvo device info + + /* + * Only parse SDVO mappings when the general definitions block child + * device size matches that of the *legacy* child device config + * struct. Thus, SDVO mapping will be skipped for newer VBT. */ - if (p_defs->child_dev_size != sizeof(*p_child)) { - /* different child dev size . Ignore it */ - DRM_DEBUG_KMS("different child size is found. Invalid.\n"); + if (p_defs->child_dev_size != sizeof(*child)) { + DRM_DEBUG_KMS("Unsupported child device size for SDVO mapping.\n"); return; } /* get the block size of general definitions */ @@ -487,37 +434,37 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv, p_defs->child_dev_size; count = 0; for (i = 0; i < child_device_num; i++) { - p_child = child_device_ptr(p_defs, i); - if (!p_child->old.device_type) { + child = &child_device_ptr(p_defs, i)->old; + if (!child->device_type) { /* skip the device block if device type is invalid */ continue; } - if (p_child->old.slave_addr != SLAVE_ADDR1 && - p_child->old.slave_addr != SLAVE_ADDR2) { + if (child->slave_addr != SLAVE_ADDR1 && + child->slave_addr != SLAVE_ADDR2) { /* * If the slave address is neither 0x70 nor 0x72, * it is not a SDVO device. Skip it. */ continue; } - if (p_child->old.dvo_port != DEVICE_PORT_DVOB && - p_child->old.dvo_port != DEVICE_PORT_DVOC) { + if (child->dvo_port != DEVICE_PORT_DVOB && + child->dvo_port != DEVICE_PORT_DVOC) { /* skip the incorrect SDVO port */ DRM_DEBUG_KMS("Incorrect SDVO port. Skip it\n"); continue; } DRM_DEBUG_KMS("the SDVO device with slave addr %2x is found on" - " %s port\n", - p_child->old.slave_addr, - (p_child->old.dvo_port == DEVICE_PORT_DVOB) ? - "SDVOB" : "SDVOC"); - p_mapping = &(dev_priv->sdvo_mappings[p_child->old.dvo_port - 1]); + " %s port\n", + child->slave_addr, + (child->dvo_port == DEVICE_PORT_DVOB) ? + "SDVOB" : "SDVOC"); + p_mapping = &(dev_priv->sdvo_mappings[child->dvo_port - 1]); if (!p_mapping->initialized) { - p_mapping->dvo_port = p_child->old.dvo_port; - p_mapping->slave_addr = p_child->old.slave_addr; - p_mapping->dvo_wiring = p_child->old.dvo_wiring; - p_mapping->ddc_pin = p_child->old.ddc_pin; - p_mapping->i2c_pin = p_child->old.i2c_pin; + p_mapping->dvo_port = child->dvo_port; + p_mapping->slave_addr = child->slave_addr; + p_mapping->dvo_wiring = child->dvo_wiring; + p_mapping->ddc_pin = child->ddc_pin; + p_mapping->i2c_pin = child->i2c_pin; p_mapping->initialized = 1; DRM_DEBUG_KMS("SDVO device: dvo=%x, addr=%x, wiring=%d, ddc_pin=%d, i2c_pin=%d\n", p_mapping->dvo_port, @@ -529,7 +476,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *dev_priv, DRM_DEBUG_KMS("Maybe one SDVO port is shared by " "two SDVO device.\n"); } - if (p_child->old.slave2_addr) { + if (child->slave2_addr) { /* Maybe this is a SDVO device with multiple inputs */ /* And the mapping info is not added */ DRM_DEBUG_KMS("there exists the slave2_addr. Maybe this" @@ -859,6 +806,12 @@ parse_mipi(struct drm_i915_private *dev_priv, const struct bdb_header *bdb) return; } + /* Fail gracefully for forward incompatible sequence block. */ + if (sequence->version >= 3) { + DRM_ERROR("Unable to parse MIPI Sequence Block v3+\n"); + return; + } + DRM_DEBUG_DRIVER("Found MIPI sequence block\n"); block_size = get_blocksize(sequence); @@ -946,6 +899,17 @@ err: memset(dev_priv->vbt.dsi.sequence, 0, sizeof(dev_priv->vbt.dsi.sequence)); } +static u8 translate_iboost(u8 val) +{ + static const u8 mapping[] = { 1, 3, 7 }; /* See VBT spec */ + + if (val >= ARRAY_SIZE(mapping)) { + DRM_DEBUG_KMS("Unsupported I_boost value found in VBT (%d), display may not work properly\n", val); + return 0; + } + return mapping[val]; +} + static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, const struct bdb_header *bdb) { @@ -954,23 +918,23 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, uint8_t hdmi_level_shift; int i, j; bool is_dvi, is_hdmi, is_dp, is_edp, is_crt; - uint8_t aux_channel; + uint8_t aux_channel, ddc_pin; /* Each DDI port can have more than one value on the "DVO Port" field, * so look for all the possible values for each port and abort if more * than one is found. */ - int dvo_ports[][2] = { - {DVO_PORT_HDMIA, DVO_PORT_DPA}, - {DVO_PORT_HDMIB, DVO_PORT_DPB}, - {DVO_PORT_HDMIC, DVO_PORT_DPC}, - {DVO_PORT_HDMID, DVO_PORT_DPD}, - {DVO_PORT_CRT, -1 /* Port E can only be DVO_PORT_CRT */ }, + int dvo_ports[][3] = { + {DVO_PORT_HDMIA, DVO_PORT_DPA, -1}, + {DVO_PORT_HDMIB, DVO_PORT_DPB, -1}, + {DVO_PORT_HDMIC, DVO_PORT_DPC, -1}, + {DVO_PORT_HDMID, DVO_PORT_DPD, -1}, + {DVO_PORT_CRT, DVO_PORT_HDMIE, DVO_PORT_DPE}, }; /* Find the child device to use, abort if more than one found. */ for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { it = dev_priv->vbt.child_dev + i; - for (j = 0; j < 2; j++) { + for (j = 0; j < 3; j++) { if (dvo_ports[port][j] == -1) break; @@ -988,6 +952,7 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, return; aux_channel = child->raw[25]; + ddc_pin = child->common.ddc_pin; is_dvi = child->common.device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING; is_dp = child->common.device_type & DEVICE_TYPE_DISPLAYPORT_OUTPUT; @@ -1019,22 +984,53 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, DRM_DEBUG_KMS("Port %c is internal DP\n", port_name(port)); if (is_dvi) { - if (child->common.ddc_pin == 0x05 && port != PORT_B) + if (port == PORT_E) { + info->alternate_ddc_pin = ddc_pin; + /* if DDIE share ddc pin with other port, then + * dvi/hdmi couldn't exist on the shared port. + * Otherwise they share the same ddc bin and system + * couldn't communicate with them seperately. */ + if (ddc_pin == DDC_PIN_B) { + dev_priv->vbt.ddi_port_info[PORT_B].supports_dvi = 0; + dev_priv->vbt.ddi_port_info[PORT_B].supports_hdmi = 0; + } else if (ddc_pin == DDC_PIN_C) { + dev_priv->vbt.ddi_port_info[PORT_C].supports_dvi = 0; + dev_priv->vbt.ddi_port_info[PORT_C].supports_hdmi = 0; + } else if (ddc_pin == DDC_PIN_D) { + dev_priv->vbt.ddi_port_info[PORT_D].supports_dvi = 0; + dev_priv->vbt.ddi_port_info[PORT_D].supports_hdmi = 0; + } + } else if (ddc_pin == DDC_PIN_B && port != PORT_B) DRM_DEBUG_KMS("Unexpected DDC pin for port B\n"); - if (child->common.ddc_pin == 0x04 && port != PORT_C) + else if (ddc_pin == DDC_PIN_C && port != PORT_C) DRM_DEBUG_KMS("Unexpected DDC pin for port C\n"); - if (child->common.ddc_pin == 0x06 && port != PORT_D) + else if (ddc_pin == DDC_PIN_D && port != PORT_D) DRM_DEBUG_KMS("Unexpected DDC pin for port D\n"); } if (is_dp) { - if (aux_channel == 0x40 && port != PORT_A) + if (port == PORT_E) { + info->alternate_aux_channel = aux_channel; + /* if DDIE share aux channel with other port, then + * DP couldn't exist on the shared port. Otherwise + * they share the same aux channel and system + * couldn't communicate with them seperately. */ + if (aux_channel == DP_AUX_A) + dev_priv->vbt.ddi_port_info[PORT_A].supports_dp = 0; + else if (aux_channel == DP_AUX_B) + dev_priv->vbt.ddi_port_info[PORT_B].supports_dp = 0; + else if (aux_channel == DP_AUX_C) + dev_priv->vbt.ddi_port_info[PORT_C].supports_dp = 0; + else if (aux_channel == DP_AUX_D) + dev_priv->vbt.ddi_port_info[PORT_D].supports_dp = 0; + } + else if (aux_channel == DP_AUX_A && port != PORT_A) DRM_DEBUG_KMS("Unexpected AUX channel for port A\n"); - if (aux_channel == 0x10 && port != PORT_B) + else if (aux_channel == DP_AUX_B && port != PORT_B) DRM_DEBUG_KMS("Unexpected AUX channel for port B\n"); - if (aux_channel == 0x20 && port != PORT_C) + else if (aux_channel == DP_AUX_C && port != PORT_C) DRM_DEBUG_KMS("Unexpected AUX channel for port C\n"); - if (aux_channel == 0x30 && port != PORT_D) + else if (aux_channel == DP_AUX_D && port != PORT_D) DRM_DEBUG_KMS("Unexpected AUX channel for port D\n"); } @@ -1046,6 +1042,16 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, hdmi_level_shift); info->hdmi_level_shift = hdmi_level_shift; } + + /* Parse the I_boost config for SKL and above */ + if (bdb->version >= 196 && (child->common.flags_1 & IBOOST_ENABLE)) { + info->dp_boost_level = translate_iboost(child->common.iboost_level & 0xF); + DRM_DEBUG_KMS("VBT (e)DP boost level for port %c: %d\n", + port_name(port), info->dp_boost_level); + info->hdmi_boost_level = translate_iboost(child->common.iboost_level >> 4); + DRM_DEBUG_KMS("VBT HDMI boost level for port %c: %d\n", + port_name(port), info->hdmi_boost_level); + } } static void parse_ddi_ports(struct drm_i915_private *dev_priv, @@ -1075,17 +1081,39 @@ parse_device_mapping(struct drm_i915_private *dev_priv, const union child_device_config *p_child; union child_device_config *child_dev_ptr; int i, child_device_num, count; - u16 block_size; + u8 expected_size; + u16 block_size; p_defs = find_section(bdb, BDB_GENERAL_DEFINITIONS); if (!p_defs) { DRM_DEBUG_KMS("No general definition block is found, no devices defined.\n"); return; } - if (p_defs->child_dev_size < sizeof(*p_child)) { - DRM_ERROR("General definiton block child device size is too small.\n"); + if (bdb->version < 195) { + expected_size = sizeof(struct old_child_dev_config); + } else if (bdb->version == 195) { + expected_size = 37; + } else if (bdb->version <= 197) { + expected_size = 38; + } else { + expected_size = 38; + BUILD_BUG_ON(sizeof(*p_child) < 38); + DRM_DEBUG_DRIVER("Expected child device config size for VBT version %u not known; assuming %u\n", + bdb->version, expected_size); + } + + /* The legacy sized child device config is the minimum we need. */ + if (p_defs->child_dev_size < sizeof(struct old_child_dev_config)) { + DRM_ERROR("Child device config size %u is too small.\n", + p_defs->child_dev_size); return; } + + /* Flag an error for unexpected size, but continue anyway. */ + if (p_defs->child_dev_size != expected_size) + DRM_ERROR("Unexpected child device config size %u (expected %u for VBT version %u)\n", + p_defs->child_dev_size, expected_size, bdb->version); + /* get the block size of general definitions */ block_size = get_blocksize(p_defs); /* get the number of child device */ @@ -1130,7 +1158,14 @@ parse_device_mapping(struct drm_i915_private *dev_priv, child_dev_ptr = dev_priv->vbt.child_dev + count; count++; - memcpy(child_dev_ptr, p_child, sizeof(*p_child)); + + /* + * Copy as much as we know (sizeof) and is available + * (child_dev_size) of the child device. Accessing the data must + * depend on VBT version. + */ + memcpy(child_dev_ptr, p_child, + min_t(size_t, p_defs->child_dev_size, sizeof(*p_child))); } return; } @@ -1199,20 +1234,13 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = { { } }; -static const struct bdb_header *validate_vbt(const void __iomem *_base, +static const struct bdb_header *validate_vbt(const void *base, size_t size, - const void __iomem *_vbt, + const void *_vbt, const char *source) { - /* - * This is the one place where we explicitly discard the address space - * (__iomem) of the BIOS/VBT. (And this will cause a sparse complaint.) - * From now on everything is based on 'base', and treated as regular - * memory. - */ - const void *base = (const void *) _base; - size_t offset = _vbt - _base; - const struct vbt_header *vbt = base + offset; + size_t offset = _vbt - base; + const struct vbt_header *vbt = _vbt; const struct bdb_header *bdb; if (offset + sizeof(struct vbt_header) > size) { @@ -1250,7 +1278,15 @@ static const struct bdb_header *find_vbt(void __iomem *bios, size_t size) /* Scour memory looking for the VBT signature. */ for (i = 0; i + 4 < size; i++) { if (ioread32(bios + i) == *((const u32 *) "$VBT")) { - bdb = validate_vbt(bios, size, bios + i, "PCI ROM"); + /* + * This is the one place where we explicitly discard the + * address space (__iomem) of the BIOS/VBT. From now on + * everything is based on 'base', and treated as regular + * memory. + */ + void *_bios = (void __force *) bios; + + bdb = validate_vbt(_bios, size, _bios + i, "PCI ROM"); break; } } @@ -1318,21 +1354,3 @@ intel_parse_bios(struct drm_device *dev) return 0; } - -/* Ensure that vital registers have been initialised, even if the BIOS - * is absent or just failing to do its job. - */ -void intel_setup_bios(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - /* Set the Panel Power On/Off timings if uninitialized. */ - if (!HAS_PCH_SPLIT(dev) && - I915_READ(PP_ON_DELAYS) == 0 && I915_READ(PP_OFF_DELAYS) == 0) { - /* Set T2 to 40ms and T5 to 200ms */ - I915_WRITE(PP_ON_DELAYS, 0x019007d0); - - /* Set T3 to 35ms and Tx to 200ms */ - I915_WRITE(PP_OFF_DELAYS, 0x015e07d0); - } -} diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h index af0b47652752..7ec8c9aefb84 100644 --- a/drivers/gpu/drm/i915/intel_bios.h +++ b/drivers/gpu/drm/i915/intel_bios.h @@ -203,9 +203,11 @@ struct bdb_general_features { #define DEVICE_PORT_DVOB 0x01 #define DEVICE_PORT_DVOC 0x02 -/* We used to keep this struct but without any version control. We should avoid +/* + * We used to keep this struct but without any version control. We should avoid * using it in the future, but it should be safe to keep using it in the old - * code. */ + * code. Do not change; we rely on its size. + */ struct old_child_dev_config { u16 handle; u16 device_type; @@ -231,6 +233,10 @@ struct old_child_dev_config { /* This one contains field offsets that are known to be common for all BDB * versions. Notice that the meaning of the contents contents may still change, * but at least the offsets are consistent. */ + +/* Definitions for flags_1 */ +#define IBOOST_ENABLE (1<<3) + struct common_child_dev_config { u16 handle; u16 device_type; @@ -239,8 +245,13 @@ struct common_child_dev_config { u8 not_common2[2]; u8 ddc_pin; u16 edid_ptr; + u8 obsolete; + u8 flags_1; + u8 not_common3[13]; + u8 iboost_level; } __packed; + /* This field changes depending on the BDB version, so the most reliable way to * read it is by checking the BDB version and reading the raw pointer. */ union child_device_config { @@ -577,7 +588,6 @@ struct bdb_psr { struct psr_table psr_table[16]; } __packed; -void intel_setup_bios(struct drm_device *dev); int intel_parse_bios(struct drm_device *dev); /* @@ -731,7 +741,6 @@ int intel_parse_bios(struct drm_device *dev); */ #define DEVICE_TYPE_eDP_BITS \ (DEVICE_TYPE_INTERNAL_CONNECTOR | \ - DEVICE_TYPE_NOT_HDMI_OUTPUT | \ DEVICE_TYPE_MIPI_OUTPUT | \ DEVICE_TYPE_COMPOSITE_OUTPUT | \ DEVICE_TYPE_DUAL_CHANNEL | \ @@ -739,7 +748,6 @@ int intel_parse_bios(struct drm_device *dev); DEVICE_TYPE_TMDS_DVI_SIGNALING | \ DEVICE_TYPE_VIDEO_SIGNALING | \ DEVICE_TYPE_DISPLAYPORT_OUTPUT | \ - DEVICE_TYPE_DIGITAL_OUTPUT | \ DEVICE_TYPE_ANALOG_OUTPUT) /* define the DVO port for HDMI output type */ @@ -747,11 +755,6 @@ int intel_parse_bios(struct drm_device *dev); #define DVO_C 2 #define DVO_D 3 -/* define the PORT for DP output type */ -#define PORT_IDPB 7 -#define PORT_IDPC 8 -#define PORT_IDPD 9 - /* Possible values for the "DVO Port" field for versions >= 155: */ #define DVO_PORT_HDMIA 0 #define DVO_PORT_HDMIB 1 @@ -764,6 +767,8 @@ int intel_parse_bios(struct drm_device *dev); #define DVO_PORT_DPC 8 #define DVO_PORT_DPD 9 #define DVO_PORT_DPA 10 +#define DVO_PORT_DPE 11 +#define DVO_PORT_HDMIE 12 #define DVO_PORT_MIPIA 21 #define DVO_PORT_MIPIB 22 #define DVO_PORT_MIPIC 23 @@ -778,6 +783,13 @@ int intel_parse_bios(struct drm_device *dev); #define MIPI_DSI_UNDEFINED_PANEL_ID 0 #define MIPI_DSI_GENERIC_PANEL_ID 1 +/* + * PMIC vs SoC Backlight support specified in pwm_blc + * field in mipi_config block below. +*/ +#define PPS_BLC_PMIC 0 +#define PPS_BLC_SOC 1 + struct mipi_config { u16 panel_id; diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index 521af2c069cb..9285fc1e64ee 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -50,7 +50,7 @@ struct intel_crt { * encoder's enable/disable callbacks */ struct intel_connector *connector; bool force_hotplug_required; - u32 adpa_reg; + i915_reg_t adpa_reg; }; static struct intel_crt *intel_encoder_to_crt(struct intel_encoder *encoder) @@ -138,18 +138,6 @@ static void hsw_crt_get_config(struct intel_encoder *encoder, pipe_config->base.adjusted_mode.flags |= intel_crt_get_flags(encoder); } -static void hsw_crt_pre_enable(struct intel_encoder *encoder) -{ - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - - WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL already enabled\n"); - I915_WRITE(SPLL_CTL, - SPLL_PLL_ENABLE | SPLL_PLL_FREQ_1350MHz | SPLL_PLL_SSC); - POSTING_READ(SPLL_CTL); - udelay(20); -} - /* Note: The caller is required to filter out dpms modes not supported by the * platform. */ static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode) @@ -158,7 +146,7 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crt *crt = intel_encoder_to_crt(encoder); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; u32 adpa; if (INTEL_INFO(dev)->gen >= 5) @@ -216,19 +204,6 @@ static void pch_post_disable_crt(struct intel_encoder *encoder) intel_disable_crt(encoder); } -static void hsw_crt_post_disable(struct intel_encoder *encoder) -{ - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t val; - - DRM_DEBUG_KMS("Disabling SPLL\n"); - val = I915_READ(SPLL_CTL); - WARN_ON(!(val & SPLL_PLL_ENABLE)); - I915_WRITE(SPLL_CTL, val & ~SPLL_PLL_ENABLE); - POSTING_READ(SPLL_CTL); -} - static void intel_enable_crt(struct intel_encoder *encoder) { struct intel_crt *crt = intel_encoder_to_crt(encoder); @@ -236,53 +211,6 @@ static void intel_enable_crt(struct intel_encoder *encoder) intel_crt_set_dpms(encoder, crt->connector->base.dpms); } -/* Special dpms function to support cloning between dvo/sdvo/crt. */ -static void intel_crt_dpms(struct drm_connector *connector, int mode) -{ - struct drm_device *dev = connector->dev; - struct intel_encoder *encoder = intel_attached_encoder(connector); - struct drm_crtc *crtc; - int old_dpms; - - /* PCH platforms and VLV only support on/off. */ - if (INTEL_INFO(dev)->gen >= 5 && mode != DRM_MODE_DPMS_ON) - mode = DRM_MODE_DPMS_OFF; - - if (mode == connector->dpms) - return; - - old_dpms = connector->dpms; - connector->dpms = mode; - - /* Only need to change hw state when actually enabled */ - crtc = encoder->base.crtc; - if (!crtc) { - encoder->connectors_active = false; - return; - } - - /* We need the pipe to run for anything but OFF. */ - if (mode == DRM_MODE_DPMS_OFF) - encoder->connectors_active = false; - else - encoder->connectors_active = true; - - /* We call connector dpms manually below in case pipe dpms doesn't - * change due to cloning. */ - if (mode < old_dpms) { - /* From off to on, enable the pipe first. */ - intel_crtc_update_dpms(crtc); - - intel_crt_set_dpms(encoder, mode); - } else { - intel_crt_set_dpms(encoder, mode); - - intel_crtc_update_dpms(crtc); - } - - intel_modeset_check_state(connector->dev); -} - static enum drm_mode_status intel_crt_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) @@ -327,6 +255,10 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder, if (HAS_DDI(dev)) { pipe_config->ddi_pll_sel = PORT_CLK_SEL_SPLL; pipe_config->port_clock = 135000 * 2; + + pipe_config->dpll_hw_state.wrpll = 0; + pipe_config->dpll_hw_state.spll = + SPLL_PLL_ENABLE | SPLL_PLL_FREQ_1350MHz | SPLL_PLL_SSC; } return true; @@ -423,7 +355,7 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector) { struct drm_device *dev = connector->dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 hotplug_en, orig, stat; + u32 stat; bool ret = false; int i, tries = 0; @@ -442,12 +374,12 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector) tries = 2; else tries = 1; - hotplug_en = orig = I915_READ(PORT_HOTPLUG_EN); - hotplug_en |= CRT_HOTPLUG_FORCE_DETECT; for (i = 0; i < tries ; i++) { /* turn on the FORCE_DETECT */ - I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); + i915_hotplug_interrupt_update(dev_priv, + CRT_HOTPLUG_FORCE_DETECT, + CRT_HOTPLUG_FORCE_DETECT); /* wait for FORCE_DETECT to go off */ if (wait_for((I915_READ(PORT_HOTPLUG_EN) & CRT_HOTPLUG_FORCE_DETECT) == 0, @@ -462,8 +394,7 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector) /* clear the interrupt we just generated, if any */ I915_WRITE(PORT_HOTPLUG_STAT, CRT_HOTPLUG_INT_STATUS); - /* and put the bits back */ - I915_WRITE(PORT_HOTPLUG_EN, orig); + i915_hotplug_interrupt_update(dev_priv, CRT_HOTPLUG_FORCE_DETECT, 0); return ret; } @@ -549,12 +480,8 @@ intel_crt_load_detect(struct intel_crt *crt) uint32_t vsample; uint32_t vblank, vblank_start, vblank_end; uint32_t dsl; - uint32_t bclrpat_reg; - uint32_t vtotal_reg; - uint32_t vblank_reg; - uint32_t vsync_reg; - uint32_t pipeconf_reg; - uint32_t pipe_dsl_reg; + i915_reg_t bclrpat_reg, vtotal_reg, + vblank_reg, vsync_reg, pipeconf_reg, pipe_dsl_reg; uint8_t st00; enum drm_connector_status status; @@ -587,7 +514,7 @@ intel_crt_load_detect(struct intel_crt *crt) /* Wait for next Vblank to substitue * border color for Color info */ intel_wait_for_vblank(dev, pipe); - st00 = I915_READ8(VGA_MSR_WRITE); + st00 = I915_READ8(_VGA_MSR_WRITE); status = ((st00 & (1 << 4)) != 0) ? connector_status_connected : connector_status_disconnected; @@ -632,7 +559,7 @@ intel_crt_load_detect(struct intel_crt *crt) do { count++; /* Read the ST00 VGA status register */ - st00 = I915_READ8(VGA_MSR_WRITE); + st00 = I915_READ8(_VGA_MSR_WRITE); if (st00 & (1 << 4)) detect++; } while ((I915_READ(pipe_dsl_reg) == dsl)); @@ -798,7 +725,7 @@ static void intel_crt_reset(struct drm_connector *connector) static const struct drm_connector_funcs intel_crt_connector_funcs = { .reset = intel_crt_reset, - .dpms = intel_crt_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_crt_detect, .fill_modes = drm_helper_probe_single_connector_modes, .destroy = intel_crt_destroy, @@ -850,11 +777,37 @@ void intel_crt_init(struct drm_device *dev) struct intel_crt *crt; struct intel_connector *intel_connector; struct drm_i915_private *dev_priv = dev->dev_private; + i915_reg_t adpa_reg; + u32 adpa; /* Skip machines without VGA that falsely report hotplug events */ if (dmi_check_system(intel_no_crt)) return; + if (HAS_PCH_SPLIT(dev)) + adpa_reg = PCH_ADPA; + else if (IS_VALLEYVIEW(dev)) + adpa_reg = VLV_ADPA; + else + adpa_reg = ADPA; + + adpa = I915_READ(adpa_reg); + if ((adpa & ADPA_DAC_ENABLE) == 0) { + /* + * On some machines (some IVB at least) CRT can be + * fused off, but there's no known fuse bit to + * indicate that. On these machine the ADPA register + * works normally, except the DAC enable bit won't + * take. So the only way to tell is attempt to enable + * it and see what happens. + */ + I915_WRITE(adpa_reg, adpa | ADPA_DAC_ENABLE | + ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE); + if ((I915_READ(adpa_reg) & ADPA_DAC_ENABLE) == 0) + return; + I915_WRITE(adpa_reg, adpa); + } + crt = kzalloc(sizeof(struct intel_crt), GFP_KERNEL); if (!crt) return; @@ -871,7 +824,7 @@ void intel_crt_init(struct drm_device *dev) &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA); drm_encoder_init(dev, &crt->base.base, &intel_crt_enc_funcs, - DRM_MODE_ENCODER_DAC); + DRM_MODE_ENCODER_DAC, NULL); intel_connector_attach_encoder(intel_connector, &crt->base); @@ -888,12 +841,7 @@ void intel_crt_init(struct drm_device *dev) connector->interlace_allowed = 1; connector->doublescan_allowed = 0; - if (HAS_PCH_SPLIT(dev)) - crt->adpa_reg = PCH_ADPA; - else if (IS_VALLEYVIEW(dev)) - crt->adpa_reg = VLV_ADPA; - else - crt->adpa_reg = ADPA; + crt->adpa_reg = adpa_reg; crt->base.compute_config = intel_crt_compute_config; if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev)) { @@ -908,8 +856,6 @@ void intel_crt_init(struct drm_device *dev) if (HAS_DDI(dev)) { crt->base.get_config = hsw_crt_get_config; crt->base.get_hw_state = intel_ddi_get_hw_state; - crt->base.pre_enable = hsw_crt_pre_enable; - crt->base.post_disable = hsw_crt_post_disable; } else { crt->base.get_config = intel_crt_get_config; crt->base.get_hw_state = intel_crt_get_hw_state; @@ -938,7 +884,7 @@ void intel_crt_init(struct drm_device *dev) u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT | FDI_RX_LINK_REVERSAL_OVERRIDE; - dev_priv->fdi_rx_config = I915_READ(_FDI_RXA_CTL) & fdi_config; + dev_priv->fdi_rx_config = I915_READ(FDI_RX_CTL(PIPE_A)) & fdi_config; } intel_crt_reset(connector); diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c index bcb41e61877d..6c6a6695e99c 100644 --- a/drivers/gpu/drm/i915/intel_csr.c +++ b/drivers/gpu/drm/i915/intel_csr.c @@ -42,24 +42,15 @@ */ #define I915_CSR_SKL "i915/skl_dmc_ver1.bin" +#define I915_CSR_BXT "i915/bxt_dmc_ver1.bin" MODULE_FIRMWARE(I915_CSR_SKL); +MODULE_FIRMWARE(I915_CSR_BXT); + +#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 23) -/* -* SKL CSR registers for DC5 and DC6 -*/ -#define CSR_PROGRAM_BASE 0x80000 -#define CSR_SSP_BASE_ADDR_GEN9 0x00002FC0 -#define CSR_HTP_ADDR_SKL 0x00500034 -#define CSR_SSP_BASE 0x8F074 -#define CSR_HTP_SKL 0x8F004 -#define CSR_LAST_WRITE 0x8F034 -#define CSR_LAST_WRITE_VALUE 0xc003b400 -/* MMIO address range for CSR program (0x80000 - 0x82FFF) */ #define CSR_MAX_FW_SIZE 0x2FFF #define CSR_DEFAULT_FW_OFFSET 0xFFFFFFFF -#define CSR_MMIO_START_RANGE 0x80000 -#define CSR_MMIO_END_RANGE 0x8FFFF struct intel_css_header { /* 0x09 for DMC */ @@ -176,147 +167,134 @@ struct stepping_info { }; static const struct stepping_info skl_stepping_info[] = { - {'A', '0'}, {'B', '0'}, {'C', '0'}, - {'D', '0'}, {'E', '0'}, {'F', '0'}, - {'G', '0'}, {'H', '0'}, {'I', '0'} + {'A', '0'}, {'B', '0'}, {'C', '0'}, + {'D', '0'}, {'E', '0'}, {'F', '0'}, + {'G', '0'}, {'H', '0'}, {'I', '0'} }; -static char intel_get_stepping(struct drm_device *dev) -{ - if (IS_SKYLAKE(dev) && (dev->pdev->revision < - ARRAY_SIZE(skl_stepping_info))) - return skl_stepping_info[dev->pdev->revision].stepping; - else - return -ENODATA; -} +static const struct stepping_info bxt_stepping_info[] = { + {'A', '0'}, {'A', '1'}, {'A', '2'}, + {'B', '0'}, {'B', '1'}, {'B', '2'} +}; -static char intel_get_substepping(struct drm_device *dev) +static const struct stepping_info *intel_get_stepping_info(struct drm_device *dev) { - if (IS_SKYLAKE(dev) && (dev->pdev->revision < - ARRAY_SIZE(skl_stepping_info))) - return skl_stepping_info[dev->pdev->revision].substepping; - else - return -ENODATA; -} - -/** - * intel_csr_load_status_get() - to get firmware loading status. - * @dev_priv: i915 device. - * - * This function helps to get the firmware loading status. - * - * Return: Firmware loading status. - */ -enum csr_state intel_csr_load_status_get(struct drm_i915_private *dev_priv) -{ - enum csr_state state; - - mutex_lock(&dev_priv->csr_lock); - state = dev_priv->csr.state; - mutex_unlock(&dev_priv->csr_lock); + const struct stepping_info *si; + unsigned int size; + + if (IS_SKYLAKE(dev)) { + size = ARRAY_SIZE(skl_stepping_info); + si = skl_stepping_info; + } else if (IS_BROXTON(dev)) { + size = ARRAY_SIZE(bxt_stepping_info); + si = bxt_stepping_info; + } else { + return NULL; + } - return state; -} + if (INTEL_REVID(dev) < size) + return si + INTEL_REVID(dev); -/** - * intel_csr_load_status_set() - help to set firmware loading status. - * @dev_priv: i915 device. - * @state: enumeration of firmware loading status. - * - * Set the firmware loading status. - */ -void intel_csr_load_status_set(struct drm_i915_private *dev_priv, - enum csr_state state) -{ - mutex_lock(&dev_priv->csr_lock); - dev_priv->csr.state = state; - mutex_unlock(&dev_priv->csr_lock); + return NULL; } /** * intel_csr_load_program() - write the firmware from memory to register. - * @dev: drm device. + * @dev_priv: i915 drm device. * * CSR firmware is read from a .bin file and kept in internal memory one time. * Everytime display comes back from low power state this function is called to * copy the firmware from internal memory to registers. */ -void intel_csr_load_program(struct drm_device *dev) +void intel_csr_load_program(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - __be32 *payload = dev_priv->csr.dmc_payload; + u32 *payload = dev_priv->csr.dmc_payload; uint32_t i, fw_size; - if (!IS_GEN9(dev)) { + if (!IS_GEN9(dev_priv)) { DRM_ERROR("No CSR support available for this platform\n"); return; } - mutex_lock(&dev_priv->csr_lock); + if (!dev_priv->csr.dmc_payload) { + DRM_ERROR("Tried to program CSR with empty payload\n"); + return; + } + fw_size = dev_priv->csr.dmc_fw_size; for (i = 0; i < fw_size; i++) - I915_WRITE(CSR_PROGRAM_BASE + i * 4, - (u32 __force)payload[i]); + I915_WRITE(CSR_PROGRAM(i), payload[i]); for (i = 0; i < dev_priv->csr.mmio_count; i++) { I915_WRITE(dev_priv->csr.mmioaddr[i], - dev_priv->csr.mmiodata[i]); + dev_priv->csr.mmiodata[i]); } - - dev_priv->csr.state = FW_LOADED; - mutex_unlock(&dev_priv->csr_lock); } -static void finish_csr_load(const struct firmware *fw, void *context) +static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, + const struct firmware *fw) { - struct drm_i915_private *dev_priv = context; struct drm_device *dev = dev_priv->dev; struct intel_css_header *css_header; struct intel_package_header *package_header; struct intel_dmc_header *dmc_header; struct intel_csr *csr = &dev_priv->csr; - char stepping = intel_get_stepping(dev); - char substepping = intel_get_substepping(dev); + const struct stepping_info *stepping_info = intel_get_stepping_info(dev); + char stepping, substepping; uint32_t dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes; uint32_t i; - __be32 *dmc_payload; - bool fw_loaded = false; + uint32_t *dmc_payload; - if (!fw) { - i915_firmware_load_error_print(csr->fw_path, 0); - goto out; - } + if (!fw) + return NULL; - if ((stepping == -ENODATA) || (substepping == -ENODATA)) { + if (!stepping_info) { DRM_ERROR("Unknown stepping info, firmware loading failed\n"); - goto out; + return NULL; } + stepping = stepping_info->stepping; + substepping = stepping_info->substepping; + /* Extract CSS Header information*/ css_header = (struct intel_css_header *)fw->data; if (sizeof(struct intel_css_header) != - (css_header->header_len * 4)) { + (css_header->header_len * 4)) { DRM_ERROR("Firmware has wrong CSS header length %u bytes\n", - (css_header->header_len * 4)); - goto out; + (css_header->header_len * 4)); + return NULL; + } + + csr->version = css_header->version; + + if (IS_SKYLAKE(dev) && csr->version < SKL_CSR_VERSION_REQUIRED) { + DRM_INFO("Refusing to load old Skylake DMC firmware v%u.%u," + " please upgrade to v%u.%u or later" + " [https://01.org/linuxgraphics/intel-linux-graphics-firmwares].\n", + CSR_VERSION_MAJOR(csr->version), + CSR_VERSION_MINOR(csr->version), + CSR_VERSION_MAJOR(SKL_CSR_VERSION_REQUIRED), + CSR_VERSION_MINOR(SKL_CSR_VERSION_REQUIRED)); + return NULL; } + readcount += sizeof(struct intel_css_header); /* Extract Package Header information*/ package_header = (struct intel_package_header *) - &fw->data[readcount]; + &fw->data[readcount]; if (sizeof(struct intel_package_header) != - (package_header->header_len * 4)) { + (package_header->header_len * 4)) { DRM_ERROR("Firmware has wrong package header length %u bytes\n", - (package_header->header_len * 4)); - goto out; + (package_header->header_len * 4)); + return NULL; } readcount += sizeof(struct intel_package_header); /* Search for dmc_offset to find firware binary. */ for (i = 0; i < package_header->num_entries; i++) { if (package_header->fw_info[i].substepping == '*' && - stepping == package_header->fw_info[i].stepping) { + stepping == package_header->fw_info[i].stepping) { dmc_offset = package_header->fw_info[i].offset; break; } else if (stepping == package_header->fw_info[i].stepping && @@ -324,12 +302,12 @@ static void finish_csr_load(const struct firmware *fw, void *context) dmc_offset = package_header->fw_info[i].offset; break; } else if (package_header->fw_info[i].stepping == '*' && - package_header->fw_info[i].substepping == '*') + package_header->fw_info[i].substepping == '*') dmc_offset = package_header->fw_info[i].offset; } if (dmc_offset == CSR_DEFAULT_FW_OFFSET) { DRM_ERROR("Firmware not supported for %c stepping\n", stepping); - goto out; + return NULL; } readcount += dmc_offset; @@ -337,26 +315,26 @@ static void finish_csr_load(const struct firmware *fw, void *context) dmc_header = (struct intel_dmc_header *)&fw->data[readcount]; if (sizeof(struct intel_dmc_header) != (dmc_header->header_len)) { DRM_ERROR("Firmware has wrong dmc header length %u bytes\n", - (dmc_header->header_len)); - goto out; + (dmc_header->header_len)); + return NULL; } readcount += sizeof(struct intel_dmc_header); /* Cache the dmc header info. */ if (dmc_header->mmio_count > ARRAY_SIZE(csr->mmioaddr)) { DRM_ERROR("Firmware has wrong mmio count %u\n", - dmc_header->mmio_count); - goto out; + dmc_header->mmio_count); + return NULL; } csr->mmio_count = dmc_header->mmio_count; for (i = 0; i < dmc_header->mmio_count; i++) { - if (dmc_header->mmioaddr[i] < CSR_MMIO_START_RANGE && - dmc_header->mmioaddr[i] > CSR_MMIO_END_RANGE) { + if (dmc_header->mmioaddr[i] < CSR_MMIO_START_RANGE || + dmc_header->mmioaddr[i] > CSR_MMIO_END_RANGE) { DRM_ERROR(" Firmware has wrong mmio address 0x%x\n", - dmc_header->mmioaddr[i]); - goto out; + dmc_header->mmioaddr[i]); + return NULL; } - csr->mmioaddr[i] = dmc_header->mmioaddr[i]; + csr->mmioaddr[i] = _MMIO(dmc_header->mmioaddr[i]); csr->mmiodata[i] = dmc_header->mmiodata[i]; } @@ -364,104 +342,107 @@ static void finish_csr_load(const struct firmware *fw, void *context) nbytes = dmc_header->fw_size * 4; if (nbytes > CSR_MAX_FW_SIZE) { DRM_ERROR("CSR firmware too big (%u) bytes\n", nbytes); - goto out; + return NULL; } csr->dmc_fw_size = dmc_header->fw_size; - csr->dmc_payload = kmalloc(nbytes, GFP_KERNEL); - if (!csr->dmc_payload) { + dmc_payload = kmalloc(nbytes, GFP_KERNEL); + if (!dmc_payload) { DRM_ERROR("Memory allocation failed for dmc payload\n"); - goto out; + return NULL; } - dmc_payload = csr->dmc_payload; - for (i = 0; i < dmc_header->fw_size; i++) { - uint32_t *tmp = (u32 *)&fw->data[readcount + i * 4]; - /* - * The firmware payload is an array of 32 bit words stored in - * little-endian format in the firmware image and programmed - * as 32 bit big-endian format to memory. - */ - dmc_payload[i] = cpu_to_be32(*tmp); - } + memcpy(dmc_payload, &fw->data[readcount], nbytes); + + return dmc_payload; +} + +static void csr_load_work_fn(struct work_struct *work) +{ + struct drm_i915_private *dev_priv; + struct intel_csr *csr; + const struct firmware *fw; + int ret; + + dev_priv = container_of(work, typeof(*dev_priv), csr.work); + csr = &dev_priv->csr; + + ret = request_firmware(&fw, dev_priv->csr.fw_path, + &dev_priv->dev->pdev->dev); + if (!fw) + goto out; + + dev_priv->csr.dmc_payload = parse_csr_fw(dev_priv, fw); + if (!dev_priv->csr.dmc_payload) + goto out; /* load csr program during system boot, as needed for DC states */ - intel_csr_load_program(dev); - fw_loaded = true; + intel_csr_load_program(dev_priv); out: - if (fw_loaded) - intel_runtime_pm_put(dev_priv); - else - intel_csr_load_status_set(dev_priv, FW_FAILED); + if (dev_priv->csr.dmc_payload) { + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + + DRM_INFO("Finished loading %s (v%u.%u)\n", + dev_priv->csr.fw_path, + CSR_VERSION_MAJOR(csr->version), + CSR_VERSION_MINOR(csr->version)); + } else { + DRM_ERROR("Failed to load DMC firmware, disabling rpm\n"); + } release_firmware(fw); } /** * intel_csr_ucode_init() - initialize the firmware loading. - * @dev: drm device. + * @dev_priv: i915 drm device. * * This function is called at the time of loading the display driver to read * firmware from a .bin file and copied into a internal memory. */ -void intel_csr_ucode_init(struct drm_device *dev) +void intel_csr_ucode_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_csr *csr = &dev_priv->csr; - int ret; - if (!HAS_CSR(dev)) + INIT_WORK(&dev_priv->csr.work, csr_load_work_fn); + + if (!HAS_CSR(dev_priv)) return; - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev_priv)) csr->fw_path = I915_CSR_SKL; + else if (IS_BROXTON(dev_priv)) + csr->fw_path = I915_CSR_BXT; else { DRM_ERROR("Unexpected: no known CSR firmware for platform\n"); - intel_csr_load_status_set(dev_priv, FW_FAILED); return; } + DRM_DEBUG_KMS("Loading %s\n", csr->fw_path); + /* * Obtain a runtime pm reference, until CSR is loaded, * to avoid entering runtime-suspend. */ - intel_runtime_pm_get(dev_priv); - - /* CSR supported for platform, load firmware */ - ret = request_firmware_nowait(THIS_MODULE, true, csr->fw_path, - &dev_priv->dev->pdev->dev, - GFP_KERNEL, dev_priv, - finish_csr_load); - if (ret) { - i915_firmware_load_error_print(csr->fw_path, ret); - intel_csr_load_status_set(dev_priv, FW_FAILED); - } + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + + schedule_work(&dev_priv->csr.work); } /** * intel_csr_ucode_fini() - unload the CSR firmware. - * @dev: drm device. + * @dev_priv: i915 drm device. * * Firmmware unloading includes freeing the internal momory and reset the * firmware loading status. */ -void intel_csr_ucode_fini(struct drm_device *dev) +void intel_csr_ucode_fini(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!HAS_CSR(dev)) + if (!HAS_CSR(dev_priv)) return; - intel_csr_load_status_set(dev_priv, FW_FAILED); - kfree(dev_priv->csr.dmc_payload); -} + flush_work(&dev_priv->csr.work); -void assert_csr_loaded(struct drm_i915_private *dev_priv) -{ - WARN((intel_csr_load_status_get(dev_priv) != FW_LOADED), "CSR is not loaded.\n"); - WARN(!I915_READ(CSR_PROGRAM_BASE), - "CSR program storage start is NULL\n"); - WARN(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n"); - WARN(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n"); + kfree(dev_priv->csr.dmc_payload); } diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index cacb07b7a8f1..4afb3103eb96 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -31,6 +31,7 @@ struct ddi_buf_trans { u32 trans1; /* balance leg enable, de-emph level */ u32 trans2; /* vref sel, vswing */ + u8 i_boost; /* SKL: I_boost; valid: 0x0, 0x1, 0x3, 0x7 */ }; /* HDMI/DVI modes ignore everything but the last 2 items. So we share @@ -38,134 +39,213 @@ struct ddi_buf_trans { * automatically adapt to HDMI connections as well */ static const struct ddi_buf_trans hsw_ddi_translations_dp[] = { - { 0x00FFFFFF, 0x0006000E }, - { 0x00D75FFF, 0x0005000A }, - { 0x00C30FFF, 0x00040006 }, - { 0x80AAAFFF, 0x000B0000 }, - { 0x00FFFFFF, 0x0005000A }, - { 0x00D75FFF, 0x000C0004 }, - { 0x80C30FFF, 0x000B0000 }, - { 0x00FFFFFF, 0x00040006 }, - { 0x80D75FFF, 0x000B0000 }, + { 0x00FFFFFF, 0x0006000E, 0x0 }, + { 0x00D75FFF, 0x0005000A, 0x0 }, + { 0x00C30FFF, 0x00040006, 0x0 }, + { 0x80AAAFFF, 0x000B0000, 0x0 }, + { 0x00FFFFFF, 0x0005000A, 0x0 }, + { 0x00D75FFF, 0x000C0004, 0x0 }, + { 0x80C30FFF, 0x000B0000, 0x0 }, + { 0x00FFFFFF, 0x00040006, 0x0 }, + { 0x80D75FFF, 0x000B0000, 0x0 }, }; static const struct ddi_buf_trans hsw_ddi_translations_fdi[] = { - { 0x00FFFFFF, 0x0007000E }, - { 0x00D75FFF, 0x000F000A }, - { 0x00C30FFF, 0x00060006 }, - { 0x00AAAFFF, 0x001E0000 }, - { 0x00FFFFFF, 0x000F000A }, - { 0x00D75FFF, 0x00160004 }, - { 0x00C30FFF, 0x001E0000 }, - { 0x00FFFFFF, 0x00060006 }, - { 0x00D75FFF, 0x001E0000 }, + { 0x00FFFFFF, 0x0007000E, 0x0 }, + { 0x00D75FFF, 0x000F000A, 0x0 }, + { 0x00C30FFF, 0x00060006, 0x0 }, + { 0x00AAAFFF, 0x001E0000, 0x0 }, + { 0x00FFFFFF, 0x000F000A, 0x0 }, + { 0x00D75FFF, 0x00160004, 0x0 }, + { 0x00C30FFF, 0x001E0000, 0x0 }, + { 0x00FFFFFF, 0x00060006, 0x0 }, + { 0x00D75FFF, 0x001E0000, 0x0 }, }; static const struct ddi_buf_trans hsw_ddi_translations_hdmi[] = { /* Idx NT mV d T mV d db */ - { 0x00FFFFFF, 0x0006000E }, /* 0: 400 400 0 */ - { 0x00E79FFF, 0x000E000C }, /* 1: 400 500 2 */ - { 0x00D75FFF, 0x0005000A }, /* 2: 400 600 3.5 */ - { 0x00FFFFFF, 0x0005000A }, /* 3: 600 600 0 */ - { 0x00E79FFF, 0x001D0007 }, /* 4: 600 750 2 */ - { 0x00D75FFF, 0x000C0004 }, /* 5: 600 900 3.5 */ - { 0x00FFFFFF, 0x00040006 }, /* 6: 800 800 0 */ - { 0x80E79FFF, 0x00030002 }, /* 7: 800 1000 2 */ - { 0x00FFFFFF, 0x00140005 }, /* 8: 850 850 0 */ - { 0x00FFFFFF, 0x000C0004 }, /* 9: 900 900 0 */ - { 0x00FFFFFF, 0x001C0003 }, /* 10: 950 950 0 */ - { 0x80FFFFFF, 0x00030002 }, /* 11: 1000 1000 0 */ + { 0x00FFFFFF, 0x0006000E, 0x0 },/* 0: 400 400 0 */ + { 0x00E79FFF, 0x000E000C, 0x0 },/* 1: 400 500 2 */ + { 0x00D75FFF, 0x0005000A, 0x0 },/* 2: 400 600 3.5 */ + { 0x00FFFFFF, 0x0005000A, 0x0 },/* 3: 600 600 0 */ + { 0x00E79FFF, 0x001D0007, 0x0 },/* 4: 600 750 2 */ + { 0x00D75FFF, 0x000C0004, 0x0 },/* 5: 600 900 3.5 */ + { 0x00FFFFFF, 0x00040006, 0x0 },/* 6: 800 800 0 */ + { 0x80E79FFF, 0x00030002, 0x0 },/* 7: 800 1000 2 */ + { 0x00FFFFFF, 0x00140005, 0x0 },/* 8: 850 850 0 */ + { 0x00FFFFFF, 0x000C0004, 0x0 },/* 9: 900 900 0 */ + { 0x00FFFFFF, 0x001C0003, 0x0 },/* 10: 950 950 0 */ + { 0x80FFFFFF, 0x00030002, 0x0 },/* 11: 1000 1000 0 */ }; static const struct ddi_buf_trans bdw_ddi_translations_edp[] = { - { 0x00FFFFFF, 0x00000012 }, - { 0x00EBAFFF, 0x00020011 }, - { 0x00C71FFF, 0x0006000F }, - { 0x00AAAFFF, 0x000E000A }, - { 0x00FFFFFF, 0x00020011 }, - { 0x00DB6FFF, 0x0005000F }, - { 0x00BEEFFF, 0x000A000C }, - { 0x00FFFFFF, 0x0005000F }, - { 0x00DB6FFF, 0x000A000C }, + { 0x00FFFFFF, 0x00000012, 0x0 }, + { 0x00EBAFFF, 0x00020011, 0x0 }, + { 0x00C71FFF, 0x0006000F, 0x0 }, + { 0x00AAAFFF, 0x000E000A, 0x0 }, + { 0x00FFFFFF, 0x00020011, 0x0 }, + { 0x00DB6FFF, 0x0005000F, 0x0 }, + { 0x00BEEFFF, 0x000A000C, 0x0 }, + { 0x00FFFFFF, 0x0005000F, 0x0 }, + { 0x00DB6FFF, 0x000A000C, 0x0 }, }; static const struct ddi_buf_trans bdw_ddi_translations_dp[] = { - { 0x00FFFFFF, 0x0007000E }, - { 0x00D75FFF, 0x000E000A }, - { 0x00BEFFFF, 0x00140006 }, - { 0x80B2CFFF, 0x001B0002 }, - { 0x00FFFFFF, 0x000E000A }, - { 0x00DB6FFF, 0x00160005 }, - { 0x80C71FFF, 0x001A0002 }, - { 0x00F7DFFF, 0x00180004 }, - { 0x80D75FFF, 0x001B0002 }, + { 0x00FFFFFF, 0x0007000E, 0x0 }, + { 0x00D75FFF, 0x000E000A, 0x0 }, + { 0x00BEFFFF, 0x00140006, 0x0 }, + { 0x80B2CFFF, 0x001B0002, 0x0 }, + { 0x00FFFFFF, 0x000E000A, 0x0 }, + { 0x00DB6FFF, 0x00160005, 0x0 }, + { 0x80C71FFF, 0x001A0002, 0x0 }, + { 0x00F7DFFF, 0x00180004, 0x0 }, + { 0x80D75FFF, 0x001B0002, 0x0 }, }; static const struct ddi_buf_trans bdw_ddi_translations_fdi[] = { - { 0x00FFFFFF, 0x0001000E }, - { 0x00D75FFF, 0x0004000A }, - { 0x00C30FFF, 0x00070006 }, - { 0x00AAAFFF, 0x000C0000 }, - { 0x00FFFFFF, 0x0004000A }, - { 0x00D75FFF, 0x00090004 }, - { 0x00C30FFF, 0x000C0000 }, - { 0x00FFFFFF, 0x00070006 }, - { 0x00D75FFF, 0x000C0000 }, + { 0x00FFFFFF, 0x0001000E, 0x0 }, + { 0x00D75FFF, 0x0004000A, 0x0 }, + { 0x00C30FFF, 0x00070006, 0x0 }, + { 0x00AAAFFF, 0x000C0000, 0x0 }, + { 0x00FFFFFF, 0x0004000A, 0x0 }, + { 0x00D75FFF, 0x00090004, 0x0 }, + { 0x00C30FFF, 0x000C0000, 0x0 }, + { 0x00FFFFFF, 0x00070006, 0x0 }, + { 0x00D75FFF, 0x000C0000, 0x0 }, }; static const struct ddi_buf_trans bdw_ddi_translations_hdmi[] = { /* Idx NT mV d T mV df db */ - { 0x00FFFFFF, 0x0007000E }, /* 0: 400 400 0 */ - { 0x00D75FFF, 0x000E000A }, /* 1: 400 600 3.5 */ - { 0x00BEFFFF, 0x00140006 }, /* 2: 400 800 6 */ - { 0x00FFFFFF, 0x0009000D }, /* 3: 450 450 0 */ - { 0x00FFFFFF, 0x000E000A }, /* 4: 600 600 0 */ - { 0x00D7FFFF, 0x00140006 }, /* 5: 600 800 2.5 */ - { 0x80CB2FFF, 0x001B0002 }, /* 6: 600 1000 4.5 */ - { 0x00FFFFFF, 0x00140006 }, /* 7: 800 800 0 */ - { 0x80E79FFF, 0x001B0002 }, /* 8: 800 1000 2 */ - { 0x80FFFFFF, 0x001B0002 }, /* 9: 1000 1000 0 */ + { 0x00FFFFFF, 0x0007000E, 0x0 },/* 0: 400 400 0 */ + { 0x00D75FFF, 0x000E000A, 0x0 },/* 1: 400 600 3.5 */ + { 0x00BEFFFF, 0x00140006, 0x0 },/* 2: 400 800 6 */ + { 0x00FFFFFF, 0x0009000D, 0x0 },/* 3: 450 450 0 */ + { 0x00FFFFFF, 0x000E000A, 0x0 },/* 4: 600 600 0 */ + { 0x00D7FFFF, 0x00140006, 0x0 },/* 5: 600 800 2.5 */ + { 0x80CB2FFF, 0x001B0002, 0x0 },/* 6: 600 1000 4.5 */ + { 0x00FFFFFF, 0x00140006, 0x0 },/* 7: 800 800 0 */ + { 0x80E79FFF, 0x001B0002, 0x0 },/* 8: 800 1000 2 */ + { 0x80FFFFFF, 0x001B0002, 0x0 },/* 9: 1000 1000 0 */ }; +/* Skylake H and S */ static const struct ddi_buf_trans skl_ddi_translations_dp[] = { - { 0x00000018, 0x000000a2 }, - { 0x00004014, 0x0000009B }, - { 0x00006012, 0x00000088 }, - { 0x00008010, 0x00000087 }, - { 0x00000018, 0x0000009B }, - { 0x00004014, 0x00000088 }, - { 0x00006012, 0x00000087 }, - { 0x00000018, 0x00000088 }, - { 0x00004014, 0x00000087 }, + { 0x00002016, 0x000000A0, 0x0 }, + { 0x00005012, 0x0000009B, 0x0 }, + { 0x00007011, 0x00000088, 0x0 }, + { 0x80009010, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ + { 0x00002016, 0x0000009B, 0x0 }, + { 0x00005012, 0x00000088, 0x0 }, + { 0x80007011, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ + { 0x00002016, 0x000000DF, 0x0 }, + { 0x80005012, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ }; -/* eDP 1.4 low vswing translation parameters */ +/* Skylake U */ +static const struct ddi_buf_trans skl_u_ddi_translations_dp[] = { + { 0x0000201B, 0x000000A2, 0x0 }, + { 0x00005012, 0x00000088, 0x0 }, + { 0x00007011, 0x00000087, 0x0 }, + { 0x80009010, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ + { 0x0000201B, 0x0000009D, 0x0 }, + { 0x80005012, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ + { 0x80007011, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ + { 0x00002016, 0x00000088, 0x0 }, + { 0x80005012, 0x000000C0, 0x1 }, /* Uses I_boost level 0x1 */ +}; + +/* Skylake Y */ +static const struct ddi_buf_trans skl_y_ddi_translations_dp[] = { + { 0x00000018, 0x000000A2, 0x0 }, + { 0x00005012, 0x00000088, 0x0 }, + { 0x00007011, 0x00000087, 0x0 }, + { 0x80009010, 0x000000C0, 0x3 }, /* Uses I_boost level 0x3 */ + { 0x00000018, 0x0000009D, 0x0 }, + { 0x80005012, 0x000000C0, 0x3 }, /* Uses I_boost level 0x3 */ + { 0x80007011, 0x000000C0, 0x3 }, /* Uses I_boost level 0x3 */ + { 0x00000018, 0x00000088, 0x0 }, + { 0x80005012, 0x000000C0, 0x3 }, /* Uses I_boost level 0x3 */ +}; + +/* + * Skylake H and S + * eDP 1.4 low vswing translation parameters + */ static const struct ddi_buf_trans skl_ddi_translations_edp[] = { - { 0x00000018, 0x000000a8 }, - { 0x00002016, 0x000000ab }, - { 0x00006012, 0x000000a2 }, - { 0x00008010, 0x00000088 }, - { 0x00000018, 0x000000ab }, - { 0x00004014, 0x000000a2 }, - { 0x00006012, 0x000000a6 }, - { 0x00000018, 0x000000a2 }, - { 0x00005013, 0x0000009c }, - { 0x00000018, 0x00000088 }, + { 0x00000018, 0x000000A8, 0x0 }, + { 0x00004013, 0x000000A9, 0x0 }, + { 0x00007011, 0x000000A2, 0x0 }, + { 0x00009010, 0x0000009C, 0x0 }, + { 0x00000018, 0x000000A9, 0x0 }, + { 0x00006013, 0x000000A2, 0x0 }, + { 0x00007011, 0x000000A6, 0x0 }, + { 0x00000018, 0x000000AB, 0x0 }, + { 0x00007013, 0x0000009F, 0x0 }, + { 0x00000018, 0x000000DF, 0x0 }, }; +/* + * Skylake U + * eDP 1.4 low vswing translation parameters + */ +static const struct ddi_buf_trans skl_u_ddi_translations_edp[] = { + { 0x00000018, 0x000000A8, 0x0 }, + { 0x00004013, 0x000000A9, 0x0 }, + { 0x00007011, 0x000000A2, 0x0 }, + { 0x00009010, 0x0000009C, 0x0 }, + { 0x00000018, 0x000000A9, 0x0 }, + { 0x00006013, 0x000000A2, 0x0 }, + { 0x00007011, 0x000000A6, 0x0 }, + { 0x00002016, 0x000000AB, 0x0 }, + { 0x00005013, 0x0000009F, 0x0 }, + { 0x00000018, 0x000000DF, 0x0 }, +}; +/* + * Skylake Y + * eDP 1.4 low vswing translation parameters + */ +static const struct ddi_buf_trans skl_y_ddi_translations_edp[] = { + { 0x00000018, 0x000000A8, 0x0 }, + { 0x00004013, 0x000000AB, 0x0 }, + { 0x00007011, 0x000000A4, 0x0 }, + { 0x00009010, 0x000000DF, 0x0 }, + { 0x00000018, 0x000000AA, 0x0 }, + { 0x00006013, 0x000000A4, 0x0 }, + { 0x00007011, 0x0000009D, 0x0 }, + { 0x00000018, 0x000000A0, 0x0 }, + { 0x00006012, 0x000000DF, 0x0 }, + { 0x00000018, 0x0000008A, 0x0 }, +}; + +/* Skylake U, H and S */ static const struct ddi_buf_trans skl_ddi_translations_hdmi[] = { - { 0x00000018, 0x000000ac }, - { 0x00005012, 0x0000009d }, - { 0x00007011, 0x00000088 }, - { 0x00000018, 0x000000a1 }, - { 0x00000018, 0x00000098 }, - { 0x00004013, 0x00000088 }, - { 0x00006012, 0x00000087 }, - { 0x00000018, 0x000000df }, - { 0x00003015, 0x00000087 }, - { 0x00003015, 0x000000c7 }, - { 0x00000018, 0x000000c7 }, + { 0x00000018, 0x000000AC, 0x0 }, + { 0x00005012, 0x0000009D, 0x0 }, + { 0x00007011, 0x00000088, 0x0 }, + { 0x00000018, 0x000000A1, 0x0 }, + { 0x00000018, 0x00000098, 0x0 }, + { 0x00004013, 0x00000088, 0x0 }, + { 0x00006012, 0x00000087, 0x0 }, + { 0x00000018, 0x000000DF, 0x0 }, + { 0x00003015, 0x00000087, 0x0 }, /* Default */ + { 0x00003015, 0x000000C7, 0x0 }, + { 0x00000018, 0x000000C7, 0x0 }, +}; + +/* Skylake Y */ +static const struct ddi_buf_trans skl_y_ddi_translations_hdmi[] = { + { 0x00000018, 0x000000A1, 0x0 }, + { 0x00005012, 0x000000DF, 0x0 }, + { 0x00007011, 0x00000084, 0x0 }, + { 0x00000018, 0x000000A4, 0x0 }, + { 0x00000018, 0x0000009D, 0x0 }, + { 0x00004013, 0x00000080, 0x0 }, + { 0x00006013, 0x000000C7, 0x0 }, + { 0x00000018, 0x0000008A, 0x0 }, + { 0x00003015, 0x000000C7, 0x0 }, /* Default */ + { 0x80003015, 0x000000C7, 0x7 }, /* Uses I_boost level 0x7 */ + { 0x00000018, 0x000000C7, 0x0 }, }; struct bxt_ddi_buf_trans { @@ -176,21 +256,32 @@ struct bxt_ddi_buf_trans { bool default_index; /* true if the entry represents default value */ }; -/* BSpec does not define separate vswing/pre-emphasis values for eDP. - * Using DP values for eDP as well. - */ static const struct bxt_ddi_buf_trans bxt_ddi_translations_dp[] = { /* Idx NT mV diff db */ - { 52, 0, 0, 128, true }, /* 0: 400 0 */ - { 78, 0, 0, 85, false }, /* 1: 400 3.5 */ - { 104, 0, 0, 64, false }, /* 2: 400 6 */ - { 154, 0, 0, 43, false }, /* 3: 400 9.5 */ - { 77, 0, 0, 128, false }, /* 4: 600 0 */ - { 116, 0, 0, 85, false }, /* 5: 600 3.5 */ - { 154, 0, 0, 64, false }, /* 6: 600 6 */ - { 102, 0, 0, 128, false }, /* 7: 800 0 */ - { 154, 0, 0, 85, false }, /* 8: 800 3.5 */ - { 154, 0x9A, 1, 128, false }, /* 9: 1200 0 */ + { 52, 0x9A, 0, 128, true }, /* 0: 400 0 */ + { 78, 0x9A, 0, 85, false }, /* 1: 400 3.5 */ + { 104, 0x9A, 0, 64, false }, /* 2: 400 6 */ + { 154, 0x9A, 0, 43, false }, /* 3: 400 9.5 */ + { 77, 0x9A, 0, 128, false }, /* 4: 600 0 */ + { 116, 0x9A, 0, 85, false }, /* 5: 600 3.5 */ + { 154, 0x9A, 0, 64, false }, /* 6: 600 6 */ + { 102, 0x9A, 0, 128, false }, /* 7: 800 0 */ + { 154, 0x9A, 0, 85, false }, /* 8: 800 3.5 */ + { 154, 0x9A, 1, 128, false }, /* 9: 1200 0 */ +}; + +static const struct bxt_ddi_buf_trans bxt_ddi_translations_edp[] = { + /* Idx NT mV diff db */ + { 26, 0, 0, 128, false }, /* 0: 200 0 */ + { 38, 0, 0, 112, false }, /* 1: 200 1.5 */ + { 48, 0, 0, 96, false }, /* 2: 200 4 */ + { 54, 0, 0, 69, false }, /* 3: 200 6 */ + { 32, 0, 0, 128, false }, /* 4: 250 0 */ + { 48, 0, 0, 104, false }, /* 5: 250 1.5 */ + { 54, 0, 0, 85, false }, /* 6: 250 4 */ + { 43, 0, 0, 128, false }, /* 7: 300 0 */ + { 54, 0, 0, 101, false }, /* 8: 300 1.5 */ + { 48, 0, 0, 128, false }, /* 9: 300 0 */ }; /* BSpec has 2 recommended values - entries 0 and 8. @@ -198,38 +289,46 @@ static const struct bxt_ddi_buf_trans bxt_ddi_translations_dp[] = { */ static const struct bxt_ddi_buf_trans bxt_ddi_translations_hdmi[] = { /* Idx NT mV diff db */ - { 52, 0, 0, 128, false }, /* 0: 400 0 */ - { 52, 0, 0, 85, false }, /* 1: 400 3.5 */ - { 52, 0, 0, 64, false }, /* 2: 400 6 */ - { 42, 0, 0, 43, false }, /* 3: 400 9.5 */ - { 77, 0, 0, 128, false }, /* 4: 600 0 */ - { 77, 0, 0, 85, false }, /* 5: 600 3.5 */ - { 77, 0, 0, 64, false }, /* 6: 600 6 */ - { 102, 0, 0, 128, false }, /* 7: 800 0 */ - { 102, 0, 0, 85, false }, /* 8: 800 3.5 */ + { 52, 0x9A, 0, 128, false }, /* 0: 400 0 */ + { 52, 0x9A, 0, 85, false }, /* 1: 400 3.5 */ + { 52, 0x9A, 0, 64, false }, /* 2: 400 6 */ + { 42, 0x9A, 0, 43, false }, /* 3: 400 9.5 */ + { 77, 0x9A, 0, 128, false }, /* 4: 600 0 */ + { 77, 0x9A, 0, 85, false }, /* 5: 600 3.5 */ + { 77, 0x9A, 0, 64, false }, /* 6: 600 6 */ + { 102, 0x9A, 0, 128, false }, /* 7: 800 0 */ + { 102, 0x9A, 0, 85, false }, /* 8: 800 3.5 */ { 154, 0x9A, 1, 128, true }, /* 9: 1200 0 */ }; +static void bxt_ddi_vswing_sequence(struct drm_device *dev, u32 level, + enum port port, int type); + static void ddi_get_encoder_port(struct intel_encoder *intel_encoder, struct intel_digital_port **dig_port, enum port *port) { struct drm_encoder *encoder = &intel_encoder->base; - int type = intel_encoder->type; - if (type == INTEL_OUTPUT_DP_MST) { + switch (intel_encoder->type) { + case INTEL_OUTPUT_DP_MST: *dig_port = enc_to_mst(encoder)->primary; *port = (*dig_port)->port; - } else if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP || - type == INTEL_OUTPUT_HDMI || type == INTEL_OUTPUT_UNKNOWN) { + break; + case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_EDP: + case INTEL_OUTPUT_HDMI: + case INTEL_OUTPUT_UNKNOWN: *dig_port = enc_to_dig_port(encoder); *port = (*dig_port)->port; - } else if (type == INTEL_OUTPUT_ANALOG) { + break; + case INTEL_OUTPUT_ANALOG: *dig_port = NULL; *port = PORT_E; - } else { - DRM_ERROR("Invalid DDI encoder type %d\n", type); - BUG(); + break; + default: + WARN(1, "Invalid DDI encoder type %d\n", intel_encoder->type); + break; } } @@ -246,7 +345,78 @@ enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder) static bool intel_dig_port_supports_hdmi(const struct intel_digital_port *intel_dig_port) { - return intel_dig_port->hdmi.hdmi_reg; + return i915_mmio_reg_valid(intel_dig_port->hdmi.hdmi_reg); +} + +static const struct ddi_buf_trans *skl_get_buf_trans_dp(struct drm_device *dev, + int *n_entries) +{ + const struct ddi_buf_trans *ddi_translations; + + if (IS_SKL_ULX(dev)) { + ddi_translations = skl_y_ddi_translations_dp; + *n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp); + } else if (IS_SKL_ULT(dev)) { + ddi_translations = skl_u_ddi_translations_dp; + *n_entries = ARRAY_SIZE(skl_u_ddi_translations_dp); + } else { + ddi_translations = skl_ddi_translations_dp; + *n_entries = ARRAY_SIZE(skl_ddi_translations_dp); + } + + return ddi_translations; +} + +static const struct ddi_buf_trans *skl_get_buf_trans_edp(struct drm_device *dev, + int *n_entries) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + const struct ddi_buf_trans *ddi_translations; + + if (IS_SKL_ULX(dev)) { + if (dev_priv->edp_low_vswing) { + ddi_translations = skl_y_ddi_translations_edp; + *n_entries = ARRAY_SIZE(skl_y_ddi_translations_edp); + } else { + ddi_translations = skl_y_ddi_translations_dp; + *n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp); + } + } else if (IS_SKL_ULT(dev)) { + if (dev_priv->edp_low_vswing) { + ddi_translations = skl_u_ddi_translations_edp; + *n_entries = ARRAY_SIZE(skl_u_ddi_translations_edp); + } else { + ddi_translations = skl_u_ddi_translations_dp; + *n_entries = ARRAY_SIZE(skl_u_ddi_translations_dp); + } + } else { + if (dev_priv->edp_low_vswing) { + ddi_translations = skl_ddi_translations_edp; + *n_entries = ARRAY_SIZE(skl_ddi_translations_edp); + } else { + ddi_translations = skl_ddi_translations_dp; + *n_entries = ARRAY_SIZE(skl_ddi_translations_dp); + } + } + + return ddi_translations; +} + +static const struct ddi_buf_trans * +skl_get_buf_trans_hdmi(struct drm_device *dev, + int *n_entries) +{ + const struct ddi_buf_trans *ddi_translations; + + if (IS_SKL_ULX(dev)) { + ddi_translations = skl_y_ddi_translations_hdmi; + *n_entries = ARRAY_SIZE(skl_y_ddi_translations_hdmi); + } else { + ddi_translations = skl_ddi_translations_hdmi; + *n_entries = ARRAY_SIZE(skl_ddi_translations_hdmi); + } + + return ddi_translations; } /* @@ -260,7 +430,7 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port, bool supports_hdmi) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg; + u32 iboost_bit = 0; int i, n_hdmi_entries, n_dp_entries, n_edp_entries, hdmi_default_entry, size; int hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift; @@ -278,21 +448,19 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port, bxt_ddi_vswing_sequence(dev, hdmi_level, port, INTEL_OUTPUT_HDMI); return; - } else if (IS_SKYLAKE(dev)) { + } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { ddi_translations_fdi = NULL; - ddi_translations_dp = skl_ddi_translations_dp; - n_dp_entries = ARRAY_SIZE(skl_ddi_translations_dp); - if (dev_priv->edp_low_vswing) { - ddi_translations_edp = skl_ddi_translations_edp; - n_edp_entries = ARRAY_SIZE(skl_ddi_translations_edp); - } else { - ddi_translations_edp = skl_ddi_translations_dp; - n_edp_entries = ARRAY_SIZE(skl_ddi_translations_dp); - } - - ddi_translations_hdmi = skl_ddi_translations_hdmi; - n_hdmi_entries = ARRAY_SIZE(skl_ddi_translations_hdmi); - hdmi_default_entry = 7; + ddi_translations_dp = + skl_get_buf_trans_dp(dev, &n_dp_entries); + ddi_translations_edp = + skl_get_buf_trans_edp(dev, &n_edp_entries); + ddi_translations_hdmi = + skl_get_buf_trans_hdmi(dev, &n_hdmi_entries); + hdmi_default_entry = 8; + /* If we're boosting the current, set bit 31 of trans1 */ + if (dev_priv->vbt.ddi_port_info[port].hdmi_boost_level || + dev_priv->vbt.ddi_port_info[port].dp_boost_level) + iboost_bit = 1<<31; } else if (IS_BROADWELL(dev)) { ddi_translations_fdi = bdw_ddi_translations_fdi; ddi_translations_dp = bdw_ddi_translations_dp; @@ -352,11 +520,11 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port, BUG(); } - for (i = 0, reg = DDI_BUF_TRANS(port); i < size; i++) { - I915_WRITE(reg, ddi_translations[i].trans1); - reg += 4; - I915_WRITE(reg, ddi_translations[i].trans2); - reg += 4; + for (i = 0; i < size; i++) { + I915_WRITE(DDI_BUF_TRANS_LO(port, i), + ddi_translations[i].trans1 | iboost_bit); + I915_WRITE(DDI_BUF_TRANS_HI(port, i), + ddi_translations[i].trans2); } if (!supports_hdmi) @@ -368,10 +536,10 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port, hdmi_level = hdmi_default_entry; /* Entry 9 is for HDMI: */ - I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans1); - reg += 4; - I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans2); - reg += 4; + I915_WRITE(DDI_BUF_TRANS_LO(port, i), + ddi_translations_hdmi[hdmi_level].trans1 | iboost_bit); + I915_WRITE(DDI_BUF_TRANS_HI(port, i), + ddi_translations_hdmi[hdmi_level].trans2); } /* Program DDI buffers translations for DP. By default, program ports A-D in DP @@ -390,8 +558,10 @@ void intel_prepare_ddi(struct drm_device *dev) enum port port; bool supports_hdmi; - ddi_get_encoder_port(intel_encoder, &intel_dig_port, &port); + if (intel_encoder->type == INTEL_OUTPUT_DSI) + continue; + ddi_get_encoder_port(intel_encoder, &intel_dig_port, &port); if (visited[port]) continue; @@ -406,7 +576,7 @@ void intel_prepare_ddi(struct drm_device *dev) static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, enum port port) { - uint32_t reg = DDI_BUF_CTL(port); + i915_reg_t reg = DDI_BUF_CTL(port); int i; for (i = 0; i < 16; i++) { @@ -440,7 +610,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) * * WaFDIAutoLinkSetTimingOverrride:hsw */ - I915_WRITE(_FDI_RXA_MISC, FDI_RX_PWRDN_LANE1_VAL(2) | + I915_WRITE(FDI_RX_MISC(PIPE_A), FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2) | FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90); @@ -448,13 +618,13 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) rx_ctl_val = dev_priv->fdi_rx_config | FDI_RX_ENHANCE_FRAME_ENABLE | FDI_RX_PLL_ENABLE | FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes); - I915_WRITE(_FDI_RXA_CTL, rx_ctl_val); - POSTING_READ(_FDI_RXA_CTL); + I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); + POSTING_READ(FDI_RX_CTL(PIPE_A)); udelay(220); /* Switch from Rawclk to PCDclk */ rx_ctl_val |= FDI_PCDCLK; - I915_WRITE(_FDI_RXA_CTL, rx_ctl_val); + I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); /* Configure Port Clock Select */ I915_WRITE(PORT_CLK_SEL(PORT_E), intel_crtc->config->ddi_pll_sel); @@ -483,21 +653,21 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) udelay(600); /* Program PCH FDI Receiver TU */ - I915_WRITE(_FDI_RXA_TUSIZE1, TU_SIZE(64)); + I915_WRITE(FDI_RX_TUSIZE1(PIPE_A), TU_SIZE(64)); /* Enable PCH FDI Receiver with auto-training */ rx_ctl_val |= FDI_RX_ENABLE | FDI_LINK_TRAIN_AUTO; - I915_WRITE(_FDI_RXA_CTL, rx_ctl_val); - POSTING_READ(_FDI_RXA_CTL); + I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); + POSTING_READ(FDI_RX_CTL(PIPE_A)); /* Wait for FDI receiver lane calibration */ udelay(30); /* Unset FDI_RX_MISC pwrdn lanes */ - temp = I915_READ(_FDI_RXA_MISC); + temp = I915_READ(FDI_RX_MISC(PIPE_A)); temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK); - I915_WRITE(_FDI_RXA_MISC, temp); - POSTING_READ(_FDI_RXA_MISC); + I915_WRITE(FDI_RX_MISC(PIPE_A), temp); + POSTING_READ(FDI_RX_MISC(PIPE_A)); /* Wait for FDI auto training time */ udelay(5); @@ -531,15 +701,15 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) intel_wait_ddi_buf_idle(dev_priv, PORT_E); rx_ctl_val &= ~FDI_RX_ENABLE; - I915_WRITE(_FDI_RXA_CTL, rx_ctl_val); - POSTING_READ(_FDI_RXA_CTL); + I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val); + POSTING_READ(FDI_RX_CTL(PIPE_A)); /* Reset FDI_RX_MISC pwrdn lanes */ - temp = I915_READ(_FDI_RXA_MISC); + temp = I915_READ(FDI_RX_MISC(PIPE_A)); temp &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK); temp |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2); - I915_WRITE(_FDI_RXA_MISC, temp); - POSTING_READ(_FDI_RXA_MISC); + I915_WRITE(FDI_RX_MISC(PIPE_A), temp); + POSTING_READ(FDI_RX_MISC(PIPE_A)); } DRM_ERROR("FDI link training failed!\n"); @@ -554,7 +724,6 @@ void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder) intel_dp->DP = intel_dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0); intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count); - } static struct intel_encoder * @@ -625,11 +794,11 @@ intel_ddi_get_crtc_new_encoder(struct intel_crtc_state *crtc_state) (void) (&__a == &__b); \ __a > __b ? (__a - __b) : (__b - __a); }) -struct wrpll_rnp { +struct hsw_wrpll_rnp { unsigned p, n2, r2; }; -static unsigned wrpll_get_budget_for_freq(int clock) +static unsigned hsw_wrpll_get_budget_for_freq(int clock) { unsigned budget; @@ -703,9 +872,9 @@ static unsigned wrpll_get_budget_for_freq(int clock) return budget; } -static void wrpll_update_rnp(uint64_t freq2k, unsigned budget, - unsigned r2, unsigned n2, unsigned p, - struct wrpll_rnp *best) +static void hsw_wrpll_update_rnp(uint64_t freq2k, unsigned budget, + unsigned r2, unsigned n2, unsigned p, + struct hsw_wrpll_rnp *best) { uint64_t a, b, c, d, diff, diff_best; @@ -762,8 +931,8 @@ static void wrpll_update_rnp(uint64_t freq2k, unsigned budget, /* Otherwise a < c && b >= d, do nothing */ } -static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, - int reg) +static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, + i915_reg_t reg) { int refclk = LC_FREQ; int n, p, r; @@ -799,12 +968,12 @@ static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv, uint32_t dpll) { - uint32_t cfgcr1_reg, cfgcr2_reg; + i915_reg_t cfgcr1_reg, cfgcr2_reg; uint32_t cfgcr1_val, cfgcr2_val; uint32_t p0, p1, p2, dco_freq; - cfgcr1_reg = GET_CFG_CR1_REG(dpll); - cfgcr2_reg = GET_CFG_CR2_REG(dpll); + cfgcr1_reg = DPLL_CFGCR1(dpll); + cfgcr2_reg = DPLL_CFGCR2(dpll); cfgcr1_val = I915_READ(cfgcr1_reg); cfgcr2_val = I915_READ(cfgcr2_reg); @@ -856,6 +1025,26 @@ static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv, return dco_freq / (p0 * p1 * p2 * 5); } +static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) +{ + int dotclock; + + if (pipe_config->has_pch_encoder) + dotclock = intel_dotclock_calculate(pipe_config->port_clock, + &pipe_config->fdi_m_n); + else if (pipe_config->has_dp_encoder) + dotclock = intel_dotclock_calculate(pipe_config->port_clock, + &pipe_config->dp_m_n); + else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp == 36) + dotclock = pipe_config->port_clock * 2 / 3; + else + dotclock = pipe_config->port_clock; + + if (pipe_config->pixel_multiplier) + dotclock /= pipe_config->pixel_multiplier; + + pipe_config->base.adjusted_mode.crtc_clock = dotclock; +} static void skl_ddi_clock_get(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) @@ -902,12 +1091,7 @@ static void skl_ddi_clock_get(struct intel_encoder *encoder, pipe_config->port_clock = link_clock; - if (pipe_config->has_dp_encoder) - pipe_config->base.adjusted_mode.crtc_clock = - intel_dotclock_calculate(pipe_config->port_clock, - &pipe_config->dp_m_n); - else - pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock; + ddi_dotclock_get(pipe_config); } static void hsw_ddi_clock_get(struct intel_encoder *encoder, @@ -929,10 +1113,10 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder, link_clock = 270000; break; case PORT_CLK_SEL_WRPLL1: - link_clock = intel_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL1); + link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(0)); break; case PORT_CLK_SEL_WRPLL2: - link_clock = intel_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL2); + link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(1)); break; case PORT_CLK_SEL_SPLL: pll = I915_READ(SPLL_CTL) & SPLL_PLL_FREQ_MASK; @@ -954,23 +1138,32 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder, pipe_config->port_clock = link_clock * 2; - if (pipe_config->has_pch_encoder) - pipe_config->base.adjusted_mode.crtc_clock = - intel_dotclock_calculate(pipe_config->port_clock, - &pipe_config->fdi_m_n); - else if (pipe_config->has_dp_encoder) - pipe_config->base.adjusted_mode.crtc_clock = - intel_dotclock_calculate(pipe_config->port_clock, - &pipe_config->dp_m_n); - else - pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock; + ddi_dotclock_get(pipe_config); } static int bxt_calc_pll_link(struct drm_i915_private *dev_priv, enum intel_dpll_id dpll) { - /* FIXME formula not available in bspec */ - return 0; + struct intel_shared_dpll *pll; + struct intel_dpll_hw_state *state; + intel_clock_t clock; + + /* For DDI ports we always use a shared PLL. */ + if (WARN_ON(dpll == DPLL_ID_PRIVATE)) + return 0; + + pll = &dev_priv->shared_dplls[dpll]; + state = &pll->config.hw_state; + + clock.m1 = 2; + clock.m2 = (state->pll0 & PORT_PLL_M2_MASK) << 22; + if (state->pll3 & PORT_PLL_M2_FRAC_ENABLE) + clock.m2 |= state->pll2 & PORT_PLL_M2_FRAC_MASK; + clock.n = (state->pll1 & PORT_PLL_N_MASK) >> PORT_PLL_N_SHIFT; + clock.p1 = (state->ebb0 & PORT_PLL_P1_MASK) >> PORT_PLL_P1_SHIFT; + clock.p2 = (state->ebb0 & PORT_PLL_P2_MASK) >> PORT_PLL_P2_SHIFT; + + return chv_calc_dpll_params(100000, &clock); } static void bxt_ddi_clock_get(struct intel_encoder *encoder, @@ -980,16 +1173,9 @@ static void bxt_ddi_clock_get(struct intel_encoder *encoder, enum port port = intel_ddi_get_encoder_port(encoder); uint32_t dpll = port; - pipe_config->port_clock = - bxt_calc_pll_link(dev_priv, dpll); + pipe_config->port_clock = bxt_calc_pll_link(dev_priv, dpll); - if (pipe_config->has_dp_encoder) - pipe_config->base.adjusted_mode.crtc_clock = - intel_dotclock_calculate(pipe_config->port_clock, - &pipe_config->dp_m_n); - else - pipe_config->base.adjusted_mode.crtc_clock = - pipe_config->port_clock; + ddi_dotclock_get(pipe_config); } void intel_ddi_clock_get(struct intel_encoder *encoder, @@ -999,7 +1185,7 @@ void intel_ddi_clock_get(struct intel_encoder *encoder, if (INTEL_INFO(dev)->gen <= 8) hsw_ddi_clock_get(encoder, pipe_config); - else if (IS_SKYLAKE(dev)) + else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) skl_ddi_clock_get(encoder, pipe_config); else if (IS_BROXTON(dev)) bxt_ddi_clock_get(encoder, pipe_config); @@ -1011,12 +1197,12 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */, { uint64_t freq2k; unsigned p, n2, r2; - struct wrpll_rnp best = { 0, 0, 0 }; + struct hsw_wrpll_rnp best = { 0, 0, 0 }; unsigned budget; freq2k = clock / 100; - budget = wrpll_get_budget_for_freq(clock); + budget = hsw_wrpll_get_budget_for_freq(clock); /* Special case handling for 540 pixel clock: bypass WR PLL entirely * and directly pass the LC PLL to it. */ @@ -1060,8 +1246,8 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */, n2++) { for (p = P_MIN; p <= P_MAX; p += P_INC) - wrpll_update_rnp(freq2k, budget, - r2, n2, p, &best); + hsw_wrpll_update_rnp(freq2k, budget, + r2, n2, p, &best); } } @@ -1073,9 +1259,10 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */, static bool hsw_ddi_pll_select(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state, - struct intel_encoder *intel_encoder, - int clock) + struct intel_encoder *intel_encoder) { + int clock = crtc_state->port_clock; + if (intel_encoder->type == INTEL_OUTPUT_HDMI) { struct intel_shared_dpll *pll; uint32_t val; @@ -1100,11 +1287,119 @@ hsw_ddi_pll_select(struct intel_crtc *intel_crtc, } crtc_state->ddi_pll_sel = PORT_CLK_SEL_WRPLL(pll->id); + } else if (crtc_state->ddi_pll_sel == PORT_CLK_SEL_SPLL) { + struct drm_atomic_state *state = crtc_state->base.state; + struct intel_shared_dpll_config *spll = + &intel_atomic_get_shared_dpll_state(state)[DPLL_ID_SPLL]; + + if (spll->crtc_mask && + WARN_ON(spll->hw_state.spll != crtc_state->dpll_hw_state.spll)) + return false; + + crtc_state->shared_dpll = DPLL_ID_SPLL; + spll->hw_state.spll = crtc_state->dpll_hw_state.spll; + spll->crtc_mask |= 1 << intel_crtc->pipe; } return true; } +struct skl_wrpll_context { + uint64_t min_deviation; /* current minimal deviation */ + uint64_t central_freq; /* chosen central freq */ + uint64_t dco_freq; /* chosen dco freq */ + unsigned int p; /* chosen divider */ +}; + +static void skl_wrpll_context_init(struct skl_wrpll_context *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); + + ctx->min_deviation = U64_MAX; +} + +/* DCO freq must be within +1%/-6% of the DCO central freq */ +#define SKL_DCO_MAX_PDEVIATION 100 +#define SKL_DCO_MAX_NDEVIATION 600 + +static void skl_wrpll_try_divider(struct skl_wrpll_context *ctx, + uint64_t central_freq, + uint64_t dco_freq, + unsigned int divider) +{ + uint64_t deviation; + + deviation = div64_u64(10000 * abs_diff(dco_freq, central_freq), + central_freq); + + /* positive deviation */ + if (dco_freq >= central_freq) { + if (deviation < SKL_DCO_MAX_PDEVIATION && + deviation < ctx->min_deviation) { + ctx->min_deviation = deviation; + ctx->central_freq = central_freq; + ctx->dco_freq = dco_freq; + ctx->p = divider; + } + /* negative deviation */ + } else if (deviation < SKL_DCO_MAX_NDEVIATION && + deviation < ctx->min_deviation) { + ctx->min_deviation = deviation; + ctx->central_freq = central_freq; + ctx->dco_freq = dco_freq; + ctx->p = divider; + } +} + +static void skl_wrpll_get_multipliers(unsigned int p, + unsigned int *p0 /* out */, + unsigned int *p1 /* out */, + unsigned int *p2 /* out */) +{ + /* even dividers */ + if (p % 2 == 0) { + unsigned int half = p / 2; + + if (half == 1 || half == 2 || half == 3 || half == 5) { + *p0 = 2; + *p1 = 1; + *p2 = half; + } else if (half % 2 == 0) { + *p0 = 2; + *p1 = half / 2; + *p2 = 2; + } else if (half % 3 == 0) { + *p0 = 3; + *p1 = half / 3; + *p2 = 2; + } else if (half % 7 == 0) { + *p0 = 7; + *p1 = half / 7; + *p2 = 2; + } + } else if (p == 3 || p == 9) { /* 3, 5, 7, 9, 15, 21, 35 */ + *p0 = 3; + *p1 = 1; + *p2 = p / 3; + } else if (p == 5 || p == 7) { + *p0 = p; + *p1 = 1; + *p2 = 1; + } else if (p == 15) { + *p0 = 3; + *p1 = 1; + *p2 = 5; + } else if (p == 21) { + *p0 = 7; + *p1 = 1; + *p2 = 3; + } else if (p == 35) { + *p0 = 7; + *p1 = 1; + *p2 = 5; + } +} + struct skl_wrpll_params { uint32_t dco_fraction; uint32_t dco_integer; @@ -1115,159 +1410,154 @@ struct skl_wrpll_params { uint32_t central_freq; }; -static void -skl_ddi_calculate_wrpll(int clock /* in Hz */, - struct skl_wrpll_params *wrpll_params) +static void skl_wrpll_params_populate(struct skl_wrpll_params *params, + uint64_t afe_clock, + uint64_t central_freq, + uint32_t p0, uint32_t p1, uint32_t p2) { - uint64_t afe_clock = clock * 5; /* AFE Clock is 5x Pixel clock */ - uint64_t dco_central_freq[3] = {8400000000ULL, - 9000000000ULL, - 9600000000ULL}; - uint32_t min_dco_deviation = 400; - uint32_t min_dco_index = 3; - uint32_t P0[4] = {1, 2, 3, 7}; - uint32_t P2[4] = {1, 2, 3, 5}; - bool found = false; - uint32_t candidate_p = 0; - uint32_t candidate_p0[3] = {0}, candidate_p1[3] = {0}; - uint32_t candidate_p2[3] = {0}; - uint32_t dco_central_freq_deviation[3]; - uint32_t i, P1, k, dco_count; - bool retry_with_odd = false; uint64_t dco_freq; - /* Determine P0, P1 or P2 */ - for (dco_count = 0; dco_count < 3; dco_count++) { - found = false; - candidate_p = - div64_u64(dco_central_freq[dco_count], afe_clock); - if (retry_with_odd == false) - candidate_p = (candidate_p % 2 == 0 ? - candidate_p : candidate_p + 1); - - for (P1 = 1; P1 < candidate_p; P1++) { - for (i = 0; i < 4; i++) { - if (!(P0[i] != 1 || P1 == 1)) - continue; - - for (k = 0; k < 4; k++) { - if (P1 != 1 && P2[k] != 2) - continue; - - if (candidate_p == P0[i] * P1 * P2[k]) { - /* Found possible P0, P1, P2 */ - found = true; - candidate_p0[dco_count] = P0[i]; - candidate_p1[dco_count] = P1; - candidate_p2[dco_count] = P2[k]; - goto found; - } - - } - } - } + switch (central_freq) { + case 9600000000ULL: + params->central_freq = 0; + break; + case 9000000000ULL: + params->central_freq = 1; + break; + case 8400000000ULL: + params->central_freq = 3; + } -found: - if (found) { - dco_central_freq_deviation[dco_count] = - div64_u64(10000 * - abs_diff((candidate_p * afe_clock), - dco_central_freq[dco_count]), - dco_central_freq[dco_count]); - - if (dco_central_freq_deviation[dco_count] < - min_dco_deviation) { - min_dco_deviation = - dco_central_freq_deviation[dco_count]; - min_dco_index = dco_count; - } - } + switch (p0) { + case 1: + params->pdiv = 0; + break; + case 2: + params->pdiv = 1; + break; + case 3: + params->pdiv = 2; + break; + case 7: + params->pdiv = 4; + break; + default: + WARN(1, "Incorrect PDiv\n"); + } - if (min_dco_index > 2 && dco_count == 2) { - retry_with_odd = true; - dco_count = 0; - } + switch (p2) { + case 5: + params->kdiv = 0; + break; + case 2: + params->kdiv = 1; + break; + case 3: + params->kdiv = 2; + break; + case 1: + params->kdiv = 3; + break; + default: + WARN(1, "Incorrect KDiv\n"); } - if (min_dco_index > 2) { - WARN(1, "No valid values found for the given pixel clock\n"); - } else { - wrpll_params->central_freq = dco_central_freq[min_dco_index]; + params->qdiv_ratio = p1; + params->qdiv_mode = (params->qdiv_ratio == 1) ? 0 : 1; - switch (dco_central_freq[min_dco_index]) { - case 9600000000ULL: - wrpll_params->central_freq = 0; - break; - case 9000000000ULL: - wrpll_params->central_freq = 1; - break; - case 8400000000ULL: - wrpll_params->central_freq = 3; - } + dco_freq = p0 * p1 * p2 * afe_clock; - switch (candidate_p0[min_dco_index]) { - case 1: - wrpll_params->pdiv = 0; - break; - case 2: - wrpll_params->pdiv = 1; - break; - case 3: - wrpll_params->pdiv = 2; - break; - case 7: - wrpll_params->pdiv = 4; - break; - default: - WARN(1, "Incorrect PDiv\n"); - } + /* + * Intermediate values are in Hz. + * Divide by MHz to match bsepc + */ + params->dco_integer = div_u64(dco_freq, 24 * MHz(1)); + params->dco_fraction = + div_u64((div_u64(dco_freq, 24) - + params->dco_integer * MHz(1)) * 0x8000, MHz(1)); +} - switch (candidate_p2[min_dco_index]) { - case 5: - wrpll_params->kdiv = 0; - break; - case 2: - wrpll_params->kdiv = 1; - break; - case 3: - wrpll_params->kdiv = 2; - break; - case 1: - wrpll_params->kdiv = 3; - break; - default: - WARN(1, "Incorrect KDiv\n"); +static bool +skl_ddi_calculate_wrpll(int clock /* in Hz */, + struct skl_wrpll_params *wrpll_params) +{ + uint64_t afe_clock = clock * 5; /* AFE Clock is 5x Pixel clock */ + uint64_t dco_central_freq[3] = {8400000000ULL, + 9000000000ULL, + 9600000000ULL}; + static const int even_dividers[] = { 4, 6, 8, 10, 12, 14, 16, 18, 20, + 24, 28, 30, 32, 36, 40, 42, 44, + 48, 52, 54, 56, 60, 64, 66, 68, + 70, 72, 76, 78, 80, 84, 88, 90, + 92, 96, 98 }; + static const int odd_dividers[] = { 3, 5, 7, 9, 15, 21, 35 }; + static const struct { + const int *list; + int n_dividers; + } dividers[] = { + { even_dividers, ARRAY_SIZE(even_dividers) }, + { odd_dividers, ARRAY_SIZE(odd_dividers) }, + }; + struct skl_wrpll_context ctx; + unsigned int dco, d, i; + unsigned int p0, p1, p2; + + skl_wrpll_context_init(&ctx); + + for (d = 0; d < ARRAY_SIZE(dividers); d++) { + for (dco = 0; dco < ARRAY_SIZE(dco_central_freq); dco++) { + for (i = 0; i < dividers[d].n_dividers; i++) { + unsigned int p = dividers[d].list[i]; + uint64_t dco_freq = p * afe_clock; + + skl_wrpll_try_divider(&ctx, + dco_central_freq[dco], + dco_freq, + p); + /* + * Skip the remaining dividers if we're sure to + * have found the definitive divider, we can't + * improve a 0 deviation. + */ + if (ctx.min_deviation == 0) + goto skip_remaining_dividers; + } } - wrpll_params->qdiv_ratio = candidate_p1[min_dco_index]; - wrpll_params->qdiv_mode = - (wrpll_params->qdiv_ratio == 1) ? 0 : 1; - - dco_freq = candidate_p0[min_dco_index] * - candidate_p1[min_dco_index] * - candidate_p2[min_dco_index] * afe_clock; - +skip_remaining_dividers: /* - * Intermediate values are in Hz. - * Divide by MHz to match bsepc + * If a solution is found with an even divider, prefer + * this one. */ - wrpll_params->dco_integer = div_u64(dco_freq, (24 * MHz(1))); - wrpll_params->dco_fraction = - div_u64(((div_u64(dco_freq, 24) - - wrpll_params->dco_integer * MHz(1)) * 0x8000), MHz(1)); + if (d == 0 && ctx.p) + break; + } + if (!ctx.p) { + DRM_DEBUG_DRIVER("No valid divider found for %dHz\n", clock); + return false; } -} + /* + * gcc incorrectly analyses that these can be used without being + * initialized. To be fair, it's hard to guess. + */ + p0 = p1 = p2 = 0; + skl_wrpll_get_multipliers(ctx.p, &p0, &p1, &p2); + skl_wrpll_params_populate(wrpll_params, afe_clock, ctx.central_freq, + p0, p1, p2); + + return true; +} static bool skl_ddi_pll_select(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state, - struct intel_encoder *intel_encoder, - int clock) + struct intel_encoder *intel_encoder) { struct intel_shared_dpll *pll; uint32_t ctrl1, cfgcr1, cfgcr2; + int clock = crtc_state->port_clock; /* * See comment in intel_dpll_hw_state to understand why we always use 0 @@ -1281,7 +1571,8 @@ skl_ddi_pll_select(struct intel_crtc *intel_crtc, ctrl1 |= DPLL_CTRL1_HDMI_MODE(0); - skl_ddi_calculate_wrpll(clock * 1000, &wrpll_params); + if (!skl_ddi_calculate_wrpll(clock * 1000, &wrpll_params)) + return false; cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE | DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) | @@ -1293,17 +1584,14 @@ skl_ddi_pll_select(struct intel_crtc *intel_crtc, DPLL_CFGCR2_PDIV(wrpll_params.pdiv) | wrpll_params.central_freq; } else if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT) { - struct drm_encoder *encoder = &intel_encoder->base; - struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - - switch (intel_dp->link_bw) { - case DP_LINK_BW_1_62: + switch (crtc_state->port_clock / 2) { + case 81000: ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, 0); break; - case DP_LINK_BW_2_7: + case 135000: ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, 0); break; - case DP_LINK_BW_5_4: + case 270000: ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, 0); break; } @@ -1334,6 +1622,7 @@ skl_ddi_pll_select(struct intel_crtc *intel_crtc, /* bxt clock parameters */ struct bxt_clk_div { + int clock; uint32_t p1; uint32_t p2; uint32_t m2_int; @@ -1343,27 +1632,27 @@ struct bxt_clk_div { }; /* pre-calculated values for DP linkrates */ -static struct bxt_clk_div bxt_dp_clk_val[7] = { - /* 162 */ {4, 2, 32, 1677722, 1, 1}, - /* 270 */ {4, 1, 27, 0, 0, 1}, - /* 540 */ {2, 1, 27, 0, 0, 1}, - /* 216 */ {3, 2, 32, 1677722, 1, 1}, - /* 243 */ {4, 1, 24, 1258291, 1, 1}, - /* 324 */ {4, 1, 32, 1677722, 1, 1}, - /* 432 */ {3, 1, 32, 1677722, 1, 1} +static const struct bxt_clk_div bxt_dp_clk_val[] = { + {162000, 4, 2, 32, 1677722, 1, 1}, + {270000, 4, 1, 27, 0, 0, 1}, + {540000, 2, 1, 27, 0, 0, 1}, + {216000, 3, 2, 32, 1677722, 1, 1}, + {243000, 4, 1, 24, 1258291, 1, 1}, + {324000, 4, 1, 32, 1677722, 1, 1}, + {432000, 3, 1, 32, 1677722, 1, 1} }; static bool bxt_ddi_pll_select(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state, - struct intel_encoder *intel_encoder, - int clock) + struct intel_encoder *intel_encoder) { struct intel_shared_dpll *pll; struct bxt_clk_div clk_div = {0}; int vco = 0; uint32_t prop_coef, int_coef, gain_ctl, targ_cnt; - uint32_t dcoampovr_en_h, dco_amp, lanestagger; + uint32_t lanestagger; + int clock = crtc_state->port_clock; if (intel_encoder->type == INTEL_OUTPUT_HDMI) { intel_clock_t best_clock; @@ -1390,29 +1679,19 @@ bxt_ddi_pll_select(struct intel_crtc *intel_crtc, vco = best_clock.vco; } else if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT || intel_encoder->type == INTEL_OUTPUT_EDP) { - struct drm_encoder *encoder = &intel_encoder->base; - struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + int i; - switch (intel_dp->link_bw) { - case DP_LINK_BW_1_62: - clk_div = bxt_dp_clk_val[0]; - break; - case DP_LINK_BW_2_7: - clk_div = bxt_dp_clk_val[1]; - break; - case DP_LINK_BW_5_4: - clk_div = bxt_dp_clk_val[2]; - break; - default: - clk_div = bxt_dp_clk_val[0]; - DRM_ERROR("Unknown link rate\n"); + clk_div = bxt_dp_clk_val[0]; + for (i = 0; i < ARRAY_SIZE(bxt_dp_clk_val); ++i) { + if (bxt_dp_clk_val[i].clock == clock) { + clk_div = bxt_dp_clk_val[i]; + break; + } } vco = clock * 10 / 2 * clk_div.p1 * clk_div.p2; } - dco_amp = 15; - dcoampovr_en_h = 0; - if (vco >= 6200000 && vco <= 6480000) { + if (vco >= 6200000 && vco <= 6700000) { prop_coef = 4; int_coef = 9; gain_ctl = 3; @@ -1423,8 +1702,6 @@ bxt_ddi_pll_select(struct intel_crtc *intel_crtc, int_coef = 11; gain_ctl = 3; targ_cnt = 9; - if (vco >= 4800000 && vco < 5400000) - dcoampovr_en_h = 1; } else if (vco == 5400000) { prop_coef = 3; int_coef = 8; @@ -1466,10 +1743,13 @@ bxt_ddi_pll_select(struct intel_crtc *intel_crtc, crtc_state->dpll_hw_state.pll8 = targ_cnt; - if (dcoampovr_en_h) - crtc_state->dpll_hw_state.pll10 = PORT_PLL_DCO_AMP_OVR_EN_H; + crtc_state->dpll_hw_state.pll9 = 5 << PORT_PLL_LOCK_THRESHOLD_SHIFT; + + crtc_state->dpll_hw_state.pll10 = + PORT_PLL_DCO_AMP(PORT_PLL_DCO_AMP_DEFAULT) + | PORT_PLL_DCO_AMP_OVR_EN_H; - crtc_state->dpll_hw_state.pll10 |= PORT_PLL_DCO_AMP(dco_amp); + crtc_state->dpll_hw_state.ebb4 = PORT_PLL_10BIT_CLK_ENABLE; crtc_state->dpll_hw_state.pcsdw12 = LANESTAGGER_STRAP_OVRD | lanestagger; @@ -1500,17 +1780,16 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc, struct drm_device *dev = intel_crtc->base.dev; struct intel_encoder *intel_encoder = intel_ddi_get_crtc_new_encoder(crtc_state); - int clock = crtc_state->port_clock; - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) return skl_ddi_pll_select(intel_crtc, crtc_state, - intel_encoder, clock); + intel_encoder); else if (IS_BROXTON(dev)) return bxt_ddi_pll_select(intel_crtc, crtc_state, - intel_encoder, clock); + intel_encoder); else return hsw_ddi_pll_select(intel_crtc, crtc_state, - intel_encoder, clock); + intel_encoder); } void intel_ddi_set_pipe_settings(struct drm_crtc *crtc) @@ -1643,7 +1922,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc) } else temp |= TRANS_DDI_MODE_SELECT_DP_SST; - temp |= DDI_PORT_WIDTH(intel_dp->lane_count); + temp |= DDI_PORT_WIDTH(intel_crtc->config->lane_count); } else if (type == INTEL_OUTPUT_DP_MST) { struct intel_dp *intel_dp = &enc_to_mst(encoder)->primary->dp; @@ -1652,7 +1931,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc) } else temp |= TRANS_DDI_MODE_SELECT_DP_SST; - temp |= DDI_PORT_WIDTH(intel_dp->lane_count); + temp |= DDI_PORT_WIDTH(intel_crtc->config->lane_count); } else { WARN(1, "Invalid encoder type %d for pipe %c\n", intel_encoder->type, pipe_name(pipe)); @@ -1664,7 +1943,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc) void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv, enum transcoder cpu_transcoder) { - uint32_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder); + i915_reg_t reg = TRANS_DDI_FUNC_CTL(cpu_transcoder); uint32_t val = I915_READ(reg); val &= ~(TRANS_DDI_FUNC_ENABLE | TRANS_DDI_PORT_MASK | TRANS_DDI_DP_VC_PAYLOAD_ALLOC); @@ -1779,7 +2058,8 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc) { struct drm_crtc *crtc = &intel_crtc->base; - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); enum port port = intel_ddi_get_encoder_port(intel_encoder); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; @@ -1799,15 +2079,76 @@ void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc) TRANS_CLK_SEL_DISABLED); } -void bxt_ddi_vswing_sequence(struct drm_device *dev, u32 level, - enum port port, int type) +static void skl_ddi_set_iboost(struct drm_device *dev, u32 level, + enum port port, int type) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + const struct ddi_buf_trans *ddi_translations; + uint8_t iboost; + uint8_t dp_iboost, hdmi_iboost; + int n_entries; + u32 reg; + + /* VBT may override standard boost values */ + dp_iboost = dev_priv->vbt.ddi_port_info[port].dp_boost_level; + hdmi_iboost = dev_priv->vbt.ddi_port_info[port].hdmi_boost_level; + + if (type == INTEL_OUTPUT_DISPLAYPORT) { + if (dp_iboost) { + iboost = dp_iboost; + } else { + ddi_translations = skl_get_buf_trans_dp(dev, &n_entries); + iboost = ddi_translations[level].i_boost; + } + } else if (type == INTEL_OUTPUT_EDP) { + if (dp_iboost) { + iboost = dp_iboost; + } else { + ddi_translations = skl_get_buf_trans_edp(dev, &n_entries); + iboost = ddi_translations[level].i_boost; + } + } else if (type == INTEL_OUTPUT_HDMI) { + if (hdmi_iboost) { + iboost = hdmi_iboost; + } else { + ddi_translations = skl_get_buf_trans_hdmi(dev, &n_entries); + iboost = ddi_translations[level].i_boost; + } + } else { + return; + } + + /* Make sure that the requested I_boost is valid */ + if (iboost && iboost != 0x1 && iboost != 0x3 && iboost != 0x7) { + DRM_ERROR("Invalid I_boost value %u\n", iboost); + return; + } + + reg = I915_READ(DISPIO_CR_TX_BMU_CR0); + reg &= ~BALANCE_LEG_MASK(port); + reg &= ~(1 << (BALANCE_LEG_DISABLE_SHIFT + port)); + + if (iboost) + reg |= iboost << BALANCE_LEG_SHIFT(port); + else + reg |= 1 << (BALANCE_LEG_DISABLE_SHIFT + port); + + I915_WRITE(DISPIO_CR_TX_BMU_CR0, reg); +} + +static void bxt_ddi_vswing_sequence(struct drm_device *dev, u32 level, + enum port port, int type) { struct drm_i915_private *dev_priv = dev->dev_private; const struct bxt_ddi_buf_trans *ddi_translations; u32 n_entries, i; uint32_t val; - if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { + if (type == INTEL_OUTPUT_EDP && dev_priv->edp_low_vswing) { + n_entries = ARRAY_SIZE(bxt_ddi_translations_edp); + ddi_translations = bxt_ddi_translations_edp; + } else if (type == INTEL_OUTPUT_DISPLAYPORT + || type == INTEL_OUTPUT_EDP) { n_entries = ARRAY_SIZE(bxt_ddi_translations_dp); ddi_translations = bxt_ddi_translations_dp; } else if (type == INTEL_OUTPUT_HDMI) { @@ -1845,9 +2186,13 @@ void bxt_ddi_vswing_sequence(struct drm_device *dev, u32 level, I915_WRITE(BXT_PORT_TX_DW2_GRP(port), val); val = I915_READ(BXT_PORT_TX_DW3_LN0(port)); - val &= ~UNIQE_TRANGE_EN_METHOD; + val &= ~SCALE_DCOMP_METHOD; if (ddi_translations[level].enable) - val |= UNIQE_TRANGE_EN_METHOD; + val |= SCALE_DCOMP_METHOD; + + if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) + DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); + I915_WRITE(BXT_PORT_TX_DW3_GRP(port), val); val = I915_READ(BXT_PORT_TX_DW4_LN0(port)); @@ -1860,30 +2205,88 @@ void bxt_ddi_vswing_sequence(struct drm_device *dev, u32 level, I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val); } -static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder) +static uint32_t translate_signal_level(int signal_levels) { - struct drm_encoder *encoder = &intel_encoder->base; - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); - enum port port = intel_ddi_get_encoder_port(intel_encoder); - int type = intel_encoder->type; - int hdmi_level; + uint32_t level; - if (type == INTEL_OUTPUT_EDP) { - struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - intel_edp_panel_on(intel_dp); + switch (signal_levels) { + default: + DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level: 0x%x\n", + signal_levels); + case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: + level = 0; + break; + case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1: + level = 1; + break; + case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2: + level = 2; + break; + case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_3: + level = 3; + break; + + case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0: + level = 4; + break; + case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1: + level = 5; + break; + case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2: + level = 6; + break; + + case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0: + level = 7; + break; + case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1: + level = 8; + break; + + case DP_TRAIN_VOLTAGE_SWING_LEVEL_3 | DP_TRAIN_PRE_EMPH_LEVEL_0: + level = 9; + break; } - if (IS_SKYLAKE(dev)) { - uint32_t dpll = crtc->config->ddi_pll_sel; + return level; +} + +uint32_t ddi_signal_levels(struct intel_dp *intel_dp) +{ + struct intel_digital_port *dport = dp_to_dig_port(intel_dp); + struct drm_device *dev = dport->base.base.dev; + struct intel_encoder *encoder = &dport->base; + uint8_t train_set = intel_dp->train_set[0]; + int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | + DP_TRAIN_PRE_EMPHASIS_MASK); + enum port port = dport->port; + uint32_t level; + + level = translate_signal_level(signal_levels); + + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) + skl_ddi_set_iboost(dev, level, port, encoder->type); + else if (IS_BROXTON(dev)) + bxt_ddi_vswing_sequence(dev, level, port, encoder->type); + + return DDI_BUF_TRANS_SELECT(level); +} + +void intel_ddi_clk_select(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum port port = intel_ddi_get_encoder_port(encoder); + + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + uint32_t dpll = pipe_config->ddi_pll_sel; uint32_t val; /* * DPLL0 is used for eDP and is the only "private" DPLL (as * opposed to shared) on SKL */ - if (type == INTEL_OUTPUT_EDP) { + if (encoder->type == INTEL_OUTPUT_EDP) { WARN_ON(dpll != SKL_DPLL0); val = I915_READ(DPLL_CTRL1); @@ -1891,7 +2294,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder) val &= ~(DPLL_CTRL1_HDMI_MODE(dpll) | DPLL_CTRL1_SSC(dpll) | DPLL_CTRL1_LINK_RATE_MASK(dpll)); - val |= crtc->config->dpll_hw_state.ctrl1 << (dpll * 6); + val |= pipe_config->dpll_hw_state.ctrl1 << (dpll * 6); I915_WRITE(DPLL_CTRL1, val); POSTING_READ(DPLL_CTRL1); @@ -1907,19 +2310,38 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder) I915_WRITE(DPLL_CTRL2, val); - } else if (INTEL_INFO(dev)->gen < 9) { - WARN_ON(crtc->config->ddi_pll_sel == PORT_CLK_SEL_NONE); - I915_WRITE(PORT_CLK_SEL(port), crtc->config->ddi_pll_sel); + } else if (INTEL_INFO(dev_priv)->gen < 9) { + WARN_ON(pipe_config->ddi_pll_sel == PORT_CLK_SEL_NONE); + I915_WRITE(PORT_CLK_SEL(port), pipe_config->ddi_pll_sel); + } +} + +static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder) +{ + struct drm_encoder *encoder = &intel_encoder->base; + struct drm_device *dev = encoder->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); + enum port port = intel_ddi_get_encoder_port(intel_encoder); + int type = intel_encoder->type; + int hdmi_level; + + if (type == INTEL_OUTPUT_EDP) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + intel_edp_panel_on(intel_dp); } + intel_ddi_clk_select(intel_encoder, crtc->config); + if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + intel_dp_set_link_params(intel_dp, crtc->config); + intel_ddi_init_dp_buf_reg(intel_encoder); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); intel_dp_start_link_train(intel_dp); - intel_dp_complete_link_train(intel_dp); if (port != PORT_A || INTEL_INFO(dev)->gen >= 9) intel_dp_stop_link_train(intel_dp); } else if (type == INTEL_OUTPUT_HDMI) { @@ -1969,7 +2391,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder) intel_edp_panel_off(intel_dp); } - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) | DPLL_CTRL2_DDI_CLK_OFF(port))); else if (INTEL_INFO(dev)->gen < 9) @@ -2037,7 +2459,7 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder) } } -static void hsw_ddi_pll_enable(struct drm_i915_private *dev_priv, +static void hsw_ddi_wrpll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { I915_WRITE(WRPLL_CTL(pll->id), pll->config.hw_state.wrpll); @@ -2045,9 +2467,17 @@ static void hsw_ddi_pll_enable(struct drm_i915_private *dev_priv, udelay(20); } -static void hsw_ddi_pll_disable(struct drm_i915_private *dev_priv, +static void hsw_ddi_spll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { + I915_WRITE(SPLL_CTL, pll->config.hw_state.spll); + POSTING_READ(SPLL_CTL); + udelay(20); +} + +static void hsw_ddi_wrpll_disable(struct drm_i915_private *dev_priv, + struct intel_shared_dpll *pll) +{ uint32_t val; val = I915_READ(WRPLL_CTL(pll->id)); @@ -2055,9 +2485,19 @@ static void hsw_ddi_pll_disable(struct drm_i915_private *dev_priv, POSTING_READ(WRPLL_CTL(pll->id)); } -static bool hsw_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll, - struct intel_dpll_hw_state *hw_state) +static void hsw_ddi_spll_disable(struct drm_i915_private *dev_priv, + struct intel_shared_dpll *pll) +{ + uint32_t val; + + val = I915_READ(SPLL_CTL); + I915_WRITE(SPLL_CTL, val & ~SPLL_PLL_ENABLE); + POSTING_READ(SPLL_CTL); +} + +static bool hsw_ddi_wrpll_get_hw_state(struct drm_i915_private *dev_priv, + struct intel_shared_dpll *pll, + struct intel_dpll_hw_state *hw_state) { uint32_t val; @@ -2070,25 +2510,50 @@ static bool hsw_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, return val & WRPLL_PLL_ENABLE; } +static bool hsw_ddi_spll_get_hw_state(struct drm_i915_private *dev_priv, + struct intel_shared_dpll *pll, + struct intel_dpll_hw_state *hw_state) +{ + uint32_t val; + + if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PLLS)) + return false; + + val = I915_READ(SPLL_CTL); + hw_state->spll = val; + + return val & SPLL_PLL_ENABLE; +} + + static const char * const hsw_ddi_pll_names[] = { "WRPLL 1", "WRPLL 2", + "SPLL" }; static void hsw_shared_dplls_init(struct drm_i915_private *dev_priv) { int i; - dev_priv->num_shared_dpll = 2; + dev_priv->num_shared_dpll = 3; - for (i = 0; i < dev_priv->num_shared_dpll; i++) { + for (i = 0; i < 2; i++) { dev_priv->shared_dplls[i].id = i; dev_priv->shared_dplls[i].name = hsw_ddi_pll_names[i]; - dev_priv->shared_dplls[i].disable = hsw_ddi_pll_disable; - dev_priv->shared_dplls[i].enable = hsw_ddi_pll_enable; + dev_priv->shared_dplls[i].disable = hsw_ddi_wrpll_disable; + dev_priv->shared_dplls[i].enable = hsw_ddi_wrpll_enable; dev_priv->shared_dplls[i].get_hw_state = - hsw_ddi_pll_get_hw_state; + hsw_ddi_wrpll_get_hw_state; } + + /* SPLL is special, but needs to be initialized anyway.. */ + dev_priv->shared_dplls[i].id = i; + dev_priv->shared_dplls[i].name = hsw_ddi_pll_names[i]; + dev_priv->shared_dplls[i].disable = hsw_ddi_spll_disable; + dev_priv->shared_dplls[i].enable = hsw_ddi_spll_enable; + dev_priv->shared_dplls[i].get_hw_state = hsw_ddi_spll_get_hw_state; + } static const char * const skl_ddi_pll_names[] = { @@ -2098,7 +2563,7 @@ static const char * const skl_ddi_pll_names[] = { }; struct skl_dpll_regs { - u32 ctl, cfgcr1, cfgcr2; + i915_reg_t ctl, cfgcr1, cfgcr2; }; /* this array is indexed by the *shared* pll id */ @@ -2106,20 +2571,20 @@ static const struct skl_dpll_regs skl_dpll_regs[3] = { { /* DPLL 1 */ .ctl = LCPLL2_CTL, - .cfgcr1 = DPLL1_CFGCR1, - .cfgcr2 = DPLL1_CFGCR2, + .cfgcr1 = DPLL_CFGCR1(SKL_DPLL1), + .cfgcr2 = DPLL_CFGCR2(SKL_DPLL1), }, { /* DPLL 2 */ - .ctl = WRPLL_CTL1, - .cfgcr1 = DPLL2_CFGCR1, - .cfgcr2 = DPLL2_CFGCR2, + .ctl = WRPLL_CTL(0), + .cfgcr1 = DPLL_CFGCR1(SKL_DPLL2), + .cfgcr2 = DPLL_CFGCR2(SKL_DPLL2), }, { /* DPLL 3 */ - .ctl = WRPLL_CTL2, - .cfgcr1 = DPLL3_CFGCR1, - .cfgcr2 = DPLL3_CFGCR2, + .ctl = WRPLL_CTL(1), + .cfgcr1 = DPLL_CFGCR1(SKL_DPLL3), + .cfgcr2 = DPLL_CFGCR2(SKL_DPLL3), }, }; @@ -2404,7 +2869,7 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, temp = I915_READ(BXT_PORT_PLL(port, 9)); temp &= ~PORT_PLL_LOCK_THRESHOLD_MASK; - temp |= (5 << 1); + temp |= pll->config.hw_state.pll9; I915_WRITE(BXT_PORT_PLL(port, 9), temp); temp = I915_READ(BXT_PORT_PLL(port, 10)); @@ -2417,8 +2882,8 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, temp = I915_READ(BXT_PORT_PLL_EBB_4(port)); temp |= PORT_PLL_RECALIBRATE; I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); - /* Enable 10 bit clock */ - temp |= PORT_PLL_10BIT_CLK_ENABLE; + temp &= ~PORT_PLL_10BIT_CLK_ENABLE; + temp |= pll->config.hw_state.ebb4; I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); /* Enable PLL */ @@ -2469,23 +2934,49 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, return false; hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(port)); + hw_state->ebb0 &= PORT_PLL_P1_MASK | PORT_PLL_P2_MASK; + + hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(port)); + hw_state->ebb4 &= PORT_PLL_10BIT_CLK_ENABLE; + hw_state->pll0 = I915_READ(BXT_PORT_PLL(port, 0)); + hw_state->pll0 &= PORT_PLL_M2_MASK; + hw_state->pll1 = I915_READ(BXT_PORT_PLL(port, 1)); + hw_state->pll1 &= PORT_PLL_N_MASK; + hw_state->pll2 = I915_READ(BXT_PORT_PLL(port, 2)); + hw_state->pll2 &= PORT_PLL_M2_FRAC_MASK; + hw_state->pll3 = I915_READ(BXT_PORT_PLL(port, 3)); + hw_state->pll3 &= PORT_PLL_M2_FRAC_ENABLE; + hw_state->pll6 = I915_READ(BXT_PORT_PLL(port, 6)); + hw_state->pll6 &= PORT_PLL_PROP_COEFF_MASK | + PORT_PLL_INT_COEFF_MASK | + PORT_PLL_GAIN_CTL_MASK; + hw_state->pll8 = I915_READ(BXT_PORT_PLL(port, 8)); + hw_state->pll8 &= PORT_PLL_TARGET_CNT_MASK; + + hw_state->pll9 = I915_READ(BXT_PORT_PLL(port, 9)); + hw_state->pll9 &= PORT_PLL_LOCK_THRESHOLD_MASK; + hw_state->pll10 = I915_READ(BXT_PORT_PLL(port, 10)); + hw_state->pll10 &= PORT_PLL_DCO_AMP_OVR_EN_H | + PORT_PLL_DCO_AMP_MASK; + /* * While we write to the group register to program all lanes at once we * can read only lane registers. We configure all lanes the same way, so * here just read out lanes 0/1 and output a note if lanes 2/3 differ. */ hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(port)); - if (I915_READ(BXT_PORT_PCS_DW12_LN23(port) != hw_state->pcsdw12)) + if (I915_READ(BXT_PORT_PCS_DW12_LN23(port)) != hw_state->pcsdw12) DRM_DEBUG_DRIVER("lane stagger config different for lane 01 (%08x) and 23 (%08x)\n", hw_state->pcsdw12, I915_READ(BXT_PORT_PCS_DW12_LN23(port))); + hw_state->pcsdw12 &= LANE_STAGGER_MASK | LANESTAGGER_STRAP_OVRD; return true; } @@ -2510,24 +3001,23 @@ void intel_ddi_pll_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; uint32_t val = I915_READ(LCPLL_CTL); - int cdclk_freq; - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) skl_shared_dplls_init(dev_priv); else if (IS_BROXTON(dev)) bxt_shared_dplls_init(dev_priv); else hsw_shared_dplls_init(dev_priv); - cdclk_freq = dev_priv->display.get_display_clock_speed(dev); - DRM_DEBUG_KMS("CDCLK running at %dKHz\n", cdclk_freq); + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + int cdclk_freq; - if (IS_SKYLAKE(dev)) { + cdclk_freq = dev_priv->display.get_display_clock_speed(dev); dev_priv->skl_boot_cdclk = cdclk_freq; + if (skl_sanitize_cdclk(dev_priv)) + DRM_DEBUG_KMS("Sanitized cdclk programmed by pre-os\n"); if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) DRM_ERROR("LCPLL1 is disabled\n"); - else - intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS); } else if (IS_BROXTON(dev)) { broxton_init_cdclk(dev); broxton_ddi_phy_init(dev); @@ -2546,11 +3036,11 @@ void intel_ddi_pll_init(struct drm_device *dev) } } -void intel_ddi_prepare_link_retrain(struct drm_encoder *encoder) +void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) { - struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); - struct intel_dp *intel_dp = &intel_dig_port->dp; - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = + to_i915(intel_dig_port->base.base.dev); enum port port = intel_dig_port->port; uint32_t val; bool wait = false; @@ -2600,36 +3090,22 @@ void intel_ddi_fdi_disable(struct drm_crtc *crtc) intel_ddi_post_disable(intel_encoder); - val = I915_READ(_FDI_RXA_CTL); + val = I915_READ(FDI_RX_CTL(PIPE_A)); val &= ~FDI_RX_ENABLE; - I915_WRITE(_FDI_RXA_CTL, val); + I915_WRITE(FDI_RX_CTL(PIPE_A), val); - val = I915_READ(_FDI_RXA_MISC); + val = I915_READ(FDI_RX_MISC(PIPE_A)); val &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK); val |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2); - I915_WRITE(_FDI_RXA_MISC, val); + I915_WRITE(FDI_RX_MISC(PIPE_A), val); - val = I915_READ(_FDI_RXA_CTL); + val = I915_READ(FDI_RX_CTL(PIPE_A)); val &= ~FDI_PCDCLK; - I915_WRITE(_FDI_RXA_CTL, val); + I915_WRITE(FDI_RX_CTL(PIPE_A), val); - val = I915_READ(_FDI_RXA_CTL); + val = I915_READ(FDI_RX_CTL(PIPE_A)); val &= ~FDI_RX_PLL_ENABLE; - I915_WRITE(_FDI_RXA_CTL, val); -} - -static void intel_ddi_hot_plug(struct intel_encoder *intel_encoder) -{ - struct intel_digital_port *intel_dig_port = enc_to_dig_port(&intel_encoder->base); - int type = intel_dig_port->base.type; - - if (type != INTEL_OUTPUT_DISPLAYPORT && - type != INTEL_OUTPUT_EDP && - type != INTEL_OUTPUT_UNKNOWN) { - return; - } - - intel_dp_hot_plug(intel_encoder); + I915_WRITE(FDI_RX_CTL(PIPE_A), val); } void intel_ddi_get_config(struct intel_encoder *encoder, @@ -2675,7 +3151,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder, pipe_config->has_hdmi_sink = true; intel_hdmi = enc_to_intel_hdmi(&encoder->base); - if (intel_hdmi->infoframe_enabled(&encoder->base)) + if (intel_hdmi->infoframe_enabled(&encoder->base, pipe_config)) pipe_config->has_infoframe = true; break; case TRANS_DDI_MODE_SELECT_DVI: @@ -2684,6 +3160,8 @@ void intel_ddi_get_config(struct intel_encoder *encoder, case TRANS_DDI_MODE_SELECT_DP_SST: case TRANS_DDI_MODE_SELECT_DP_MST: pipe_config->has_dp_encoder = true; + pipe_config->lane_count = + ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1; intel_dp_get_m_n(intel_crtc, pipe_config); break; default: @@ -2793,10 +3271,9 @@ void intel_ddi_init(struct drm_device *dev, enum port port) dev_priv->vbt.ddi_port_info[port].supports_hdmi); init_dp = dev_priv->vbt.ddi_port_info[port].supports_dp; if (!init_dp && !init_hdmi) { - DRM_DEBUG_KMS("VBT says port %c is not DVI/HDMI/DP compatible, assuming it is\n", + DRM_DEBUG_KMS("VBT says port %c is not DVI/HDMI/DP compatible, respect it\n", port_name(port)); - init_hdmi = true; - init_dp = true; + return; } intel_dig_port = kzalloc(sizeof(*intel_dig_port), GFP_KERNEL); @@ -2807,7 +3284,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port) encoder = &intel_encoder->base; drm_encoder_init(dev, encoder, &intel_ddi_funcs, - DRM_MODE_ENCODER_TMDS); + DRM_MODE_ENCODER_TMDS, NULL); intel_encoder->compute_config = intel_ddi_compute_config; intel_encoder->enable = intel_enable_ddi; @@ -2822,17 +3299,37 @@ void intel_ddi_init(struct drm_device *dev, enum port port) (DDI_BUF_PORT_REVERSAL | DDI_A_4_LANES); + /* + * Bspec says that DDI_A_4_LANES is the only supported configuration + * for Broxton. Yet some BIOS fail to set this bit on port A if eDP + * wasn't lit up at boot. Force this bit on in our internal + * configuration so that we use the proper lane count for our + * calculations. + */ + if (IS_BROXTON(dev) && port == PORT_A) { + if (!(intel_dig_port->saved_port_bits & DDI_A_4_LANES)) { + DRM_DEBUG_KMS("BXT BIOS forgot to set DDI_A_4_LANES for port A; fixing\n"); + intel_dig_port->saved_port_bits |= DDI_A_4_LANES; + } + } + intel_encoder->type = INTEL_OUTPUT_UNKNOWN; intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); intel_encoder->cloneable = 0; - intel_encoder->hot_plug = intel_ddi_hot_plug; if (init_dp) { if (!intel_ddi_init_dp_connector(intel_dig_port)) goto err; intel_dig_port->hpd_pulse = intel_dp_hpd_pulse; - dev_priv->hpd_irq_port[port] = intel_dig_port; + /* + * On BXT A0/A1, sw needs to activate DDIA HPD logic and + * interrupts to check the external panel connection. + */ + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1) && port == PORT_B) + dev_priv->hotplug.irq_port[PORT_A] = intel_dig_port; + else + dev_priv->hotplug.irq_port[port] = intel_dig_port; } /* In theory we don't need the encoder->type check, but leave it just in diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 87476ff181dd..bda6b9c82e66 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -44,6 +44,8 @@ #include <drm/drm_plane_helper.h> #include <drm/drm_rect.h> #include <linux/dma_remapping.h> +#include <linux/reservation.h> +#include <linux/dma-buf.h> /* Primary plane formats for gen <= 3 */ static const uint32_t i8xx_primary_formats[] = { @@ -72,6 +74,10 @@ static const uint32_t skl_primary_formats[] = { DRM_FORMAT_ABGR8888, DRM_FORMAT_XRGB2101010, DRM_FORMAT_XBGR2101010, + DRM_FORMAT_YUYV, + DRM_FORMAT_YVYU, + DRM_FORMAT_UYVY, + DRM_FORMAT_VYUY, }; /* Cursor formats */ @@ -86,9 +92,6 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, static void ironlake_pch_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); -static int intel_set_mode(struct drm_crtc *crtc, - struct drm_atomic_state *state, - bool force_restore); static int intel_framebuffer_init(struct drm_device *dev, struct intel_framebuffer *ifb, struct drm_mode_fb_cmd2 *mode_cmd, @@ -105,22 +108,16 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config); static void chv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config); -static void intel_begin_crtc_commit(struct drm_crtc *crtc); -static void intel_finish_crtc_commit(struct drm_crtc *crtc); +static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); +static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state); static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state, int num_connectors); -static void intel_crtc_enable_planes(struct drm_crtc *crtc); -static void intel_crtc_disable_planes(struct drm_crtc *crtc); - -static struct intel_encoder *intel_find_encoder(struct intel_connector *connector, int pipe) -{ - if (!connector->mst_port) - return connector->encoder; - else - return &connector->mst_port->mst_encoders[pipe]->base; -} +static void skylake_pfit_enable(struct intel_crtc *crtc); +static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force); +static void ironlake_pfit_enable(struct intel_crtc *crtc); +static void intel_modeset_setup_hw_state(struct drm_device *dev); typedef struct { int min, max; @@ -137,6 +134,42 @@ struct intel_limit { intel_p2_t p2; }; +/* returns HPLL frequency in kHz */ +static int valleyview_get_vco(struct drm_i915_private *dev_priv) +{ + int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 }; + + /* Obtain SKU information */ + mutex_lock(&dev_priv->sb_lock); + hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) & + CCK_FUSE_HPLL_FREQ_MASK; + mutex_unlock(&dev_priv->sb_lock); + + return vco_freq[hpll_freq] * 1000; +} + +static int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv, + const char *name, u32 reg) +{ + u32 val; + int divider; + + if (dev_priv->hpll_freq == 0) + dev_priv->hpll_freq = valleyview_get_vco(dev_priv); + + mutex_lock(&dev_priv->sb_lock); + val = vlv_cck_read(dev_priv, reg); + mutex_unlock(&dev_priv->sb_lock); + + divider = val & CCK_FREQUENCY_VALUES; + + WARN((val & CCK_FREQUENCY_STATUS) != + (divider << CCK_FREQUENCY_STATUS_SHIFT), + "%s change in progress\n", name); + + return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1); +} + int intel_pch_rawclk(struct drm_device *dev) { @@ -147,6 +180,50 @@ intel_pch_rawclk(struct drm_device *dev) return I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK; } +/* hrawclock is 1/4 the FSB frequency */ +int intel_hrawclk(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t clkcfg; + + /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */ + if (IS_VALLEYVIEW(dev)) + return 200; + + clkcfg = I915_READ(CLKCFG); + switch (clkcfg & CLKCFG_FSB_MASK) { + case CLKCFG_FSB_400: + return 100; + case CLKCFG_FSB_533: + return 133; + case CLKCFG_FSB_667: + return 166; + case CLKCFG_FSB_800: + return 200; + case CLKCFG_FSB_1067: + return 266; + case CLKCFG_FSB_1333: + return 333; + /* these two are just a guess; one of them might be right */ + case CLKCFG_FSB_1600: + case CLKCFG_FSB_1600_ALT: + return 400; + default: + return 133; + } +} + +static void intel_update_czclk(struct drm_i915_private *dev_priv) +{ + if (!IS_VALLEYVIEW(dev_priv)) + return; + + dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk", + CCK_CZ_CLOCK_CONTROL); + + DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq); +} + static inline u32 /* units of 100MHz */ intel_fdi_link_freq(struct drm_device *dev) { @@ -413,7 +490,7 @@ static const intel_limit_t intel_limits_chv = { static const intel_limit_t intel_limits_bxt = { /* FIXME: find real dot limits */ .dot = { .min = 0, .max = INT_MAX }, - .vco = { .min = 4800000, .max = 6480000 }, + .vco = { .min = 4800000, .max = 6700000 }, .n = { .min = 1, .max = 1 }, .m1 = { .min = 2, .max = 2 }, /* FIXME: find real m2 limits */ @@ -422,14 +499,10 @@ static const intel_limit_t intel_limits_bxt = { .p2 = { .p2_slow = 1, .p2_fast = 20 }, }; -static void vlv_clock(int refclk, intel_clock_t *clock) +static bool +needs_modeset(struct drm_crtc_state *state) { - clock->m = clock->m1 * clock->m2; - clock->p = clock->p1 * clock->p2; - if (WARN_ON(clock->n == 0 || clock->p == 0)) - return; - clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n); - clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); + return drm_atomic_crtc_needs_modeset(state); } /** @@ -561,15 +634,25 @@ intel_limit(struct intel_crtc_state *crtc_state, int refclk) return limit; } +/* + * Platform specific helpers to calculate the port PLL loopback- (clock.m), + * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast + * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic. + * The helpers' return value is the rate of the clock that is fed to the + * display engine's pipe which can be the above fast dot clock rate or a + * divided-down version of it. + */ /* m1 is reserved as 0 in Pineview, n is a ring counter */ -static void pineview_clock(int refclk, intel_clock_t *clock) +static int pnv_calc_dpll_params(int refclk, intel_clock_t *clock) { clock->m = clock->m2 + 2; clock->p = clock->p1 * clock->p2; if (WARN_ON(clock->n == 0 || clock->p == 0)) - return; + return 0; clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n); clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); + + return clock->dot; } static uint32_t i9xx_dpll_compute_m(struct dpll *dpll) @@ -577,25 +660,41 @@ static uint32_t i9xx_dpll_compute_m(struct dpll *dpll) return 5 * (dpll->m1 + 2) + (dpll->m2 + 2); } -static void i9xx_clock(int refclk, intel_clock_t *clock) +static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock) { clock->m = i9xx_dpll_compute_m(clock); clock->p = clock->p1 * clock->p2; if (WARN_ON(clock->n + 2 == 0 || clock->p == 0)) - return; + return 0; clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2); clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); + + return clock->dot; } -static void chv_clock(int refclk, intel_clock_t *clock) +static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock) { clock->m = clock->m1 * clock->m2; clock->p = clock->p1 * clock->p2; if (WARN_ON(clock->n == 0 || clock->p == 0)) - return; + return 0; + clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n); + clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); + + return clock->dot / 5; +} + +int chv_calc_dpll_params(int refclk, intel_clock_t *clock) +{ + clock->m = clock->m1 * clock->m2; + clock->p = clock->p1 * clock->p2; + if (WARN_ON(clock->n == 0 || clock->p == 0)) + return 0; clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m, clock->n << 22); clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); + + return clock->dot / 5; } #define INTELPllInvalid(s) do { /* DRM_DEBUG(s); */ return false; } while (0) @@ -639,16 +738,12 @@ static bool intel_PLL_is_valid(struct drm_device *dev, return true; } -static bool -i9xx_find_best_dpll(const intel_limit_t *limit, - struct intel_crtc_state *crtc_state, - int target, int refclk, intel_clock_t *match_clock, - intel_clock_t *best_clock) +static int +i9xx_select_p2_div(const intel_limit_t *limit, + const struct intel_crtc_state *crtc_state, + int target) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); - struct drm_device *dev = crtc->base.dev; - intel_clock_t clock; - int err = target; + struct drm_device *dev = crtc_state->base.crtc->dev; if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { /* @@ -657,18 +752,31 @@ i9xx_find_best_dpll(const intel_limit_t *limit, * single/dual channel state, if we even can. */ if (intel_is_dual_link_lvds(dev)) - clock.p2 = limit->p2.p2_fast; + return limit->p2.p2_fast; else - clock.p2 = limit->p2.p2_slow; + return limit->p2.p2_slow; } else { if (target < limit->p2.dot_limit) - clock.p2 = limit->p2.p2_slow; + return limit->p2.p2_slow; else - clock.p2 = limit->p2.p2_fast; + return limit->p2.p2_fast; } +} + +static bool +i9xx_find_best_dpll(const intel_limit_t *limit, + struct intel_crtc_state *crtc_state, + int target, int refclk, intel_clock_t *match_clock, + intel_clock_t *best_clock) +{ + struct drm_device *dev = crtc_state->base.crtc->dev; + intel_clock_t clock; + int err = target; memset(best_clock, 0, sizeof(*best_clock)); + clock.p2 = i9xx_select_p2_div(limit, crtc_state, target); + for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) { for (clock.m2 = limit->m2.min; @@ -681,7 +789,7 @@ i9xx_find_best_dpll(const intel_limit_t *limit, clock.p1 <= limit->p1.max; clock.p1++) { int this_err; - i9xx_clock(refclk, &clock); + i9xx_calc_dpll_params(refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) continue; @@ -708,30 +816,14 @@ pnv_find_best_dpll(const intel_limit_t *limit, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); - struct drm_device *dev = crtc->base.dev; + struct drm_device *dev = crtc_state->base.crtc->dev; intel_clock_t clock; int err = target; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { - /* - * For LVDS just rely on its current settings for dual-channel. - * We haven't figured out how to reliably set up different - * single/dual channel state, if we even can. - */ - if (intel_is_dual_link_lvds(dev)) - clock.p2 = limit->p2.p2_fast; - else - clock.p2 = limit->p2.p2_slow; - } else { - if (target < limit->p2.dot_limit) - clock.p2 = limit->p2.p2_slow; - else - clock.p2 = limit->p2.p2_fast; - } - memset(best_clock, 0, sizeof(*best_clock)); + clock.p2 = i9xx_select_p2_div(limit, crtc_state, target); + for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) { for (clock.m2 = limit->m2.min; @@ -742,7 +834,7 @@ pnv_find_best_dpll(const intel_limit_t *limit, clock.p1 <= limit->p1.max; clock.p1++) { int this_err; - pineview_clock(refclk, &clock); + pnv_calc_dpll_params(refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) continue; @@ -769,28 +861,17 @@ g4x_find_best_dpll(const intel_limit_t *limit, int target, int refclk, intel_clock_t *match_clock, intel_clock_t *best_clock) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); - struct drm_device *dev = crtc->base.dev; + struct drm_device *dev = crtc_state->base.crtc->dev; intel_clock_t clock; int max_n; - bool found; + bool found = false; /* approximately equals target * 0.00585 */ int err_most = (target >> 8) + (target >> 9); - found = false; - - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { - if (intel_is_dual_link_lvds(dev)) - clock.p2 = limit->p2.p2_fast; - else - clock.p2 = limit->p2.p2_slow; - } else { - if (target < limit->p2.dot_limit) - clock.p2 = limit->p2.p2_slow; - else - clock.p2 = limit->p2.p2_fast; - } memset(best_clock, 0, sizeof(*best_clock)); + + clock.p2 = i9xx_select_p2_div(limit, crtc_state, target); + max_n = limit->n.max; /* based on hardware requirement, prefer smaller n to precision */ for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) { @@ -803,7 +884,7 @@ g4x_find_best_dpll(const intel_limit_t *limit, clock.p1 >= limit->p1.min; clock.p1--) { int this_err; - i9xx_clock(refclk, &clock); + i9xx_calc_dpll_params(refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) continue; @@ -893,7 +974,7 @@ vlv_find_best_dpll(const intel_limit_t *limit, clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n, refclk * clock.m1); - vlv_clock(refclk, &clock); + vlv_calc_dpll_params(refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) @@ -956,7 +1037,7 @@ chv_find_best_dpll(const intel_limit_t *limit, clock.m2 = m2; - chv_clock(refclk, &clock); + chv_calc_dpll_params(refclk, &clock); if (!intel_PLL_is_valid(dev, limit, &clock)) continue; @@ -1016,7 +1097,7 @@ enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg = PIPEDSL(pipe); + i915_reg_t reg = PIPEDSL(pipe); u32 line1, line2; u32 line_mask; @@ -1026,7 +1107,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) line_mask = DSL_LINEMASK_GEN3; line1 = I915_READ(reg) & line_mask; - mdelay(5); + msleep(5); line2 = I915_READ(reg) & line_mask; return line1 == line2; @@ -1056,7 +1137,7 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc) enum pipe pipe = crtc->pipe; if (INTEL_INFO(dev)->gen >= 4) { - int reg = PIPECONF(cpu_transcoder); + i915_reg_t reg = PIPECONF(cpu_transcoder); /* Wait for the Pipe State to go off */ if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0, @@ -1069,51 +1150,6 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc) } } -/* - * ibx_digital_port_connected - is the specified port connected? - * @dev_priv: i915 private structure - * @port: the port to test - * - * Returns true if @port is connected, false otherwise. - */ -bool ibx_digital_port_connected(struct drm_i915_private *dev_priv, - struct intel_digital_port *port) -{ - u32 bit; - - if (HAS_PCH_IBX(dev_priv->dev)) { - switch (port->port) { - case PORT_B: - bit = SDE_PORTB_HOTPLUG; - break; - case PORT_C: - bit = SDE_PORTC_HOTPLUG; - break; - case PORT_D: - bit = SDE_PORTD_HOTPLUG; - break; - default: - return true; - } - } else { - switch (port->port) { - case PORT_B: - bit = SDE_PORTB_HOTPLUG_CPT; - break; - case PORT_C: - bit = SDE_PORTC_HOTPLUG_CPT; - break; - case PORT_D: - bit = SDE_PORTD_HOTPLUG_CPT; - break; - default: - return true; - } - } - - return I915_READ(SDEISR) & bit; -} - static const char *state_string(bool enabled) { return enabled ? "on" : "off"; @@ -1123,12 +1159,10 @@ static const char *state_string(bool enabled) void assert_pll(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { - int reg; u32 val; bool cur_state; - reg = DPLL(pipe); - val = I915_READ(reg); + val = I915_READ(DPLL(pipe)); cur_state = !!(val & DPLL_VCO_ENABLE); I915_STATE_WARN(cur_state != state, "PLL state assertion failure (expected %s, current %s)\n", @@ -1185,20 +1219,16 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv, static void assert_fdi_tx(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { - int reg; - u32 val; bool cur_state; enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, pipe); if (HAS_DDI(dev_priv->dev)) { /* DDI does not have a specific FDI_TX register */ - reg = TRANS_DDI_FUNC_CTL(cpu_transcoder); - val = I915_READ(reg); + u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); cur_state = !!(val & TRANS_DDI_FUNC_ENABLE); } else { - reg = FDI_TX_CTL(pipe); - val = I915_READ(reg); + u32 val = I915_READ(FDI_TX_CTL(pipe)); cur_state = !!(val & FDI_TX_ENABLE); } I915_STATE_WARN(cur_state != state, @@ -1211,12 +1241,10 @@ static void assert_fdi_tx(struct drm_i915_private *dev_priv, static void assert_fdi_rx(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { - int reg; u32 val; bool cur_state; - reg = FDI_RX_CTL(pipe); - val = I915_READ(reg); + val = I915_READ(FDI_RX_CTL(pipe)); cur_state = !!(val & FDI_RX_ENABLE); I915_STATE_WARN(cur_state != state, "FDI RX state assertion failure (expected %s, current %s)\n", @@ -1228,7 +1256,6 @@ static void assert_fdi_rx(struct drm_i915_private *dev_priv, static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - int reg; u32 val; /* ILK FDI PLL is always enabled */ @@ -1239,20 +1266,17 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, if (HAS_DDI(dev_priv->dev)) return; - reg = FDI_TX_CTL(pipe); - val = I915_READ(reg); + val = I915_READ(FDI_TX_CTL(pipe)); I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n"); } void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { - int reg; u32 val; bool cur_state; - reg = FDI_RX_CTL(pipe); - val = I915_READ(reg); + val = I915_READ(FDI_RX_CTL(pipe)); cur_state = !!(val & FDI_RX_PLL_ENABLE); I915_STATE_WARN(cur_state != state, "FDI RX PLL assertion failure (expected %s, current %s)\n", @@ -1263,7 +1287,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) { struct drm_device *dev = dev_priv->dev; - int pp_reg; + i915_reg_t pp_reg; u32 val; enum pipe panel_pipe = PIPE_A; bool locked = true; @@ -1308,7 +1332,7 @@ static void assert_cursor(struct drm_i915_private *dev_priv, bool cur_state; if (IS_845G(dev) || IS_I865G(dev)) - cur_state = I915_READ(_CURACNTR) & CURSOR_ENABLE; + cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE; else cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE; @@ -1322,8 +1346,6 @@ static void assert_cursor(struct drm_i915_private *dev_priv, void assert_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { - int reg; - u32 val; bool cur_state; enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, pipe); @@ -1337,8 +1359,7 @@ void assert_pipe(struct drm_i915_private *dev_priv, POWER_DOMAIN_TRANSCODER(cpu_transcoder))) { cur_state = false; } else { - reg = PIPECONF(cpu_transcoder); - val = I915_READ(reg); + u32 val = I915_READ(PIPECONF(cpu_transcoder)); cur_state = !!(val & PIPECONF_ENABLE); } @@ -1350,12 +1371,10 @@ void assert_pipe(struct drm_i915_private *dev_priv, static void assert_plane(struct drm_i915_private *dev_priv, enum plane plane, bool state) { - int reg; u32 val; bool cur_state; - reg = DSPCNTR(plane); - val = I915_READ(reg); + val = I915_READ(DSPCNTR(plane)); cur_state = !!(val & DISPLAY_PLANE_ENABLE); I915_STATE_WARN(cur_state != state, "plane %c assertion failure (expected %s, current %s)\n", @@ -1369,14 +1388,11 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { struct drm_device *dev = dev_priv->dev; - int reg, i; - u32 val; - int cur_pipe; + int i; /* Primary planes are fixed to pipes on gen4+ */ if (INTEL_INFO(dev)->gen >= 4) { - reg = DSPCNTR(pipe); - val = I915_READ(reg); + u32 val = I915_READ(DSPCNTR(pipe)); I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE, "plane %c assertion failure, should be disabled but not\n", plane_name(pipe)); @@ -1385,9 +1401,8 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv, /* Need to check both planes against the pipe */ for_each_pipe(dev_priv, i) { - reg = DSPCNTR(i); - val = I915_READ(reg); - cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> + u32 val = I915_READ(DSPCNTR(i)); + enum pipe cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> DISPPLANE_SEL_PIPE_SHIFT; I915_STATE_WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe, "plane %c assertion failure, should be off on pipe %c but is still active\n", @@ -1399,33 +1414,29 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { struct drm_device *dev = dev_priv->dev; - int reg, sprite; - u32 val; + int sprite; if (INTEL_INFO(dev)->gen >= 9) { for_each_sprite(dev_priv, pipe, sprite) { - val = I915_READ(PLANE_CTL(pipe, sprite)); + u32 val = I915_READ(PLANE_CTL(pipe, sprite)); I915_STATE_WARN(val & PLANE_CTL_ENABLE, "plane %d assertion failure, should be off on pipe %c but is still active\n", sprite, pipe_name(pipe)); } } else if (IS_VALLEYVIEW(dev)) { for_each_sprite(dev_priv, pipe, sprite) { - reg = SPCNTR(pipe, sprite); - val = I915_READ(reg); + u32 val = I915_READ(SPCNTR(pipe, sprite)); I915_STATE_WARN(val & SP_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", sprite_name(pipe, sprite), pipe_name(pipe)); } } else if (INTEL_INFO(dev)->gen >= 7) { - reg = SPRCTL(pipe); - val = I915_READ(reg); + u32 val = I915_READ(SPRCTL(pipe)); I915_STATE_WARN(val & SPRITE_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", plane_name(pipe), pipe_name(pipe)); } else if (INTEL_INFO(dev)->gen >= 5) { - reg = DVSCNTR(pipe); - val = I915_READ(reg); + u32 val = I915_READ(DVSCNTR(pipe)); I915_STATE_WARN(val & DVS_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", plane_name(pipe), pipe_name(pipe)); @@ -1454,12 +1465,10 @@ static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv) static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - int reg; u32 val; bool enabled; - reg = PCH_TRANSCONF(pipe); - val = I915_READ(reg); + val = I915_READ(PCH_TRANSCONF(pipe)); enabled = !!(val & TRANS_ENABLE); I915_STATE_WARN(enabled, "transcoder assertion failed, should be off on pipe %c but is still active\n", @@ -1473,8 +1482,7 @@ static bool dp_pipe_enabled(struct drm_i915_private *dev_priv, return false; if (HAS_PCH_CPT(dev_priv->dev)) { - u32 trans_dp_ctl_reg = TRANS_DP_CTL(pipe); - u32 trans_dp_ctl = I915_READ(trans_dp_ctl_reg); + u32 trans_dp_ctl = I915_READ(TRANS_DP_CTL(pipe)); if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel) return false; } else if (IS_CHERRYVIEW(dev_priv->dev)) { @@ -1538,12 +1546,13 @@ static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv, } static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv, - enum pipe pipe, int reg, u32 port_sel) + enum pipe pipe, i915_reg_t reg, + u32 port_sel) { u32 val = I915_READ(reg); I915_STATE_WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val), "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n", - reg, pipe_name(pipe)); + i915_mmio_reg_offset(reg), pipe_name(pipe)); I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & DP_PORT_EN) == 0 && (val & DP_PIPEB_SELECT), @@ -1551,12 +1560,12 @@ static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv, } static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv, - enum pipe pipe, int reg) + enum pipe pipe, i915_reg_t reg) { u32 val = I915_READ(reg); I915_STATE_WARN(hdmi_pipe_enabled(dev_priv, pipe, val), "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n", - reg, pipe_name(pipe)); + i915_mmio_reg_offset(reg), pipe_name(pipe)); I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & SDVO_ENABLE) == 0 && (val & SDVO_PIPE_B_SELECT), @@ -1566,21 +1575,18 @@ static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv, static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - int reg; u32 val; assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B); assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C); assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D); - reg = PCH_ADPA; - val = I915_READ(reg); + val = I915_READ(PCH_ADPA); I915_STATE_WARN(adpa_pipe_enabled(dev_priv, pipe, val), "PCH VGA enabled on transcoder %c, should be disabled\n", pipe_name(pipe)); - reg = PCH_LVDS; - val = I915_READ(reg); + val = I915_READ(PCH_LVDS); I915_STATE_WARN(lvds_pipe_enabled(dev_priv, pipe, val), "PCH LVDS enabled on transcoder %c, should be disabled\n", pipe_name(pipe)); @@ -1590,32 +1596,12 @@ static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv, assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMID); } -static void intel_init_dpio(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!IS_VALLEYVIEW(dev)) - return; - - /* - * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), - * CHV x1 PHY (DP/HDMI D) - * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) - */ - if (IS_CHERRYVIEW(dev)) { - DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; - DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; - } else { - DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; - } -} - static void vlv_enable_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - int reg = DPLL(crtc->pipe); + i915_reg_t reg = DPLL(crtc->pipe); u32 dpll = pipe_config->dpll_hw_state.dpll; assert_pipe_disabled(dev_priv, crtc->pipe); @@ -1694,7 +1680,7 @@ static int intel_num_dvo_pipes(struct drm_device *dev) int count = 0; for_each_intel_crtc(dev, crtc) - count += crtc->active && + count += crtc->base.state->active && intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO); return count; @@ -1704,7 +1690,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - int reg = DPLL(crtc->pipe); + i915_reg_t reg = DPLL(crtc->pipe); u32 dpll = crtc->config->dpll_hw_state.dpll; assert_pipe_disabled(dev_priv, crtc->pipe); @@ -1729,6 +1715,15 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE); } + /* + * Apparently we need to have VGA mode enabled prior to changing + * the P1/P2 dividers. Otherwise the DPLL will keep using the old + * dividers, even though the register value does change. + */ + I915_WRITE(reg, 0); + + I915_WRITE(reg, dpll); + /* Wait for the clocks to stabilize. */ POSTING_READ(reg); udelay(150); @@ -1775,7 +1770,7 @@ static void i9xx_disable_pll(struct intel_crtc *crtc) /* Disable DVO 2x clock on both PLLs if necessary */ if (IS_I830(dev) && intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) && - intel_num_dvo_pipes(dev) == 1) { + !intel_num_dvo_pipes(dev)) { I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE); I915_WRITE(DPLL(PIPE_A), @@ -1790,13 +1785,13 @@ static void i9xx_disable_pll(struct intel_crtc *crtc) /* Make sure the pipe isn't still relying on us */ assert_pipe_disabled(dev_priv, pipe); - I915_WRITE(DPLL(pipe), 0); + I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS); POSTING_READ(DPLL(pipe)); } static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) { - u32 val = 0; + u32 val; /* Make sure the pipe isn't still relying on us */ assert_pipe_disabled(dev_priv, pipe); @@ -1805,8 +1800,9 @@ static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) * Leave integrated clock source and reference clock enabled for pipe B. * The latter is needed for VGA hotplug / manual detection. */ + val = DPLL_VGA_MODE_DIS; if (pipe == PIPE_B) - val = DPLL_INTEGRATED_CRI_CLK_VLV | DPLL_REFA_CLK_ENABLE_VLV; + val = DPLL_INTEGRATED_CRI_CLK_VLV | DPLL_REF_CLK_ENABLE_VLV; I915_WRITE(DPLL(pipe), val); POSTING_READ(DPLL(pipe)); @@ -1821,7 +1817,8 @@ static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) assert_pipe_disabled(dev_priv, pipe); /* Set PLL en = 0 */ - val = DPLL_SSC_REF_CLOCK_CHV | DPLL_REFA_CLK_ENABLE_VLV; + val = DPLL_SSC_REF_CLK_CHV | + DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; if (pipe != PIPE_A) val |= DPLL_INTEGRATED_CRI_CLK_VLV; I915_WRITE(DPLL(pipe), val); @@ -1834,17 +1831,6 @@ static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) val &= ~DPIO_DCLKP_EN; vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val); - /* disable left/right clock distribution */ - if (pipe != PIPE_B) { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); - val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); - } else { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); - val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); - } - mutex_unlock(&dev_priv->sb_lock); } @@ -1853,7 +1839,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, unsigned int expected_mask) { u32 port_mask; - int dpll_reg; + i915_reg_t dpll_reg; switch (dport->port) { case PORT_B: @@ -1942,11 +1928,13 @@ static void intel_disable_shared_dpll(struct intel_crtc *crtc) struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc); /* PCH only available on ILK+ */ - BUG_ON(INTEL_INFO(dev)->gen < 5); - if (WARN_ON(pll == NULL)) - return; + if (INTEL_INFO(dev)->gen < 5) + return; - if (WARN_ON(pll->config.crtc_mask == 0)) + if (pll == NULL) + return; + + if (WARN_ON(!(pll->config.crtc_mask & (1 << drm_crtc_index(&crtc->base))))) return; DRM_DEBUG_KMS("disable %s (active %d, on? %d) for crtc %d\n", @@ -1976,7 +1964,8 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, struct drm_device *dev = dev_priv->dev; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - uint32_t reg, val, pipeconf_val; + i915_reg_t reg; + uint32_t val, pipeconf_val; /* PCH only available on ILK+ */ BUG_ON(!HAS_PCH_SPLIT(dev)); @@ -2004,11 +1993,15 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, if (HAS_PCH_IBX(dev_priv->dev)) { /* - * make the BPC in transcoder be consistent with - * that in pipeconf reg. + * Make the BPC in transcoder be consistent with + * that in pipeconf reg. For HDMI we must use 8bpc + * here for both 8bpc and 12bpc. */ val &= ~PIPECONF_BPC_MASK; - val |= pipeconf_val & PIPECONF_BPC_MASK; + if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI)) + val |= PIPECONF_8BPC; + else + val |= pipeconf_val & PIPECONF_BPC_MASK; } val &= ~TRANS_INTERLACE_MASK; @@ -2039,9 +2032,9 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv, assert_fdi_rx_enabled(dev_priv, TRANSCODER_A); /* Workaround: set timing override bit. */ - val = I915_READ(_TRANSA_CHICKEN2); + val = I915_READ(TRANS_CHICKEN2(PIPE_A)); val |= TRANS_CHICKEN2_TIMING_OVERRIDE; - I915_WRITE(_TRANSA_CHICKEN2, val); + I915_WRITE(TRANS_CHICKEN2(PIPE_A), val); val = TRANS_ENABLE; pipeconf_val = I915_READ(PIPECONF(cpu_transcoder)); @@ -2061,7 +2054,8 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { struct drm_device *dev = dev_priv->dev; - uint32_t reg, val; + i915_reg_t reg; + uint32_t val; /* FDI relies on the transcoder */ assert_fdi_tx_disabled(dev_priv, pipe); @@ -2078,7 +2072,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv, if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50)) DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe)); - if (!HAS_PCH_IBX(dev)) { + if (HAS_PCH_CPT(dev)) { /* Workaround: Clear the timing override chicken bit again. */ reg = TRANS_CHICKEN2(pipe); val = I915_READ(reg); @@ -2099,9 +2093,9 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv) DRM_ERROR("Failed to disable PCH transcoder\n"); /* Workaround: clear timing override bit. */ - val = I915_READ(_TRANSA_CHICKEN2); + val = I915_READ(TRANS_CHICKEN2(PIPE_A)); val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE; - I915_WRITE(_TRANSA_CHICKEN2, val); + I915_WRITE(TRANS_CHICKEN2(PIPE_A), val); } /** @@ -2116,12 +2110,13 @@ static void intel_enable_pipe(struct intel_crtc *crtc) struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe = crtc->pipe; - enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, - pipe); + enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; enum pipe pch_transcoder; - int reg; + i915_reg_t reg; u32 val; + DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe)); + assert_planes_disabled(dev_priv, pipe); assert_cursor_disabled(dev_priv, pipe); assert_sprites_disabled(dev_priv, pipe); @@ -2137,7 +2132,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc) * need the check. */ if (HAS_GMCH_DISPLAY(dev_priv->dev)) - if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI)) + if (crtc->config->has_dsi_encoder) assert_dsi_pll_enabled(dev_priv); else assert_pll_enabled(dev_priv, pipe); @@ -2178,9 +2173,11 @@ static void intel_disable_pipe(struct intel_crtc *crtc) struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; enum pipe pipe = crtc->pipe; - int reg; + i915_reg_t reg; u32 val; + DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe)); + /* * Make sure planes won't keep trying to pump pixels to us, * or we might hang the display. @@ -2211,28 +2208,6 @@ static void intel_disable_pipe(struct intel_crtc *crtc) intel_wait_for_pipe_off(crtc); } -/** - * intel_enable_primary_hw_plane - enable the primary plane on a given pipe - * @plane: plane to be enabled - * @crtc: crtc for the plane - * - * Enable @plane on @crtc, making sure that the pipe is running first. - */ -static void intel_enable_primary_hw_plane(struct drm_plane *plane, - struct drm_crtc *crtc) -{ - struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - - /* If the pipe isn't enabled, we can't pump pixels and may hang */ - assert_pipe_enabled(dev_priv, intel_crtc->pipe); - to_intel_plane_state(plane->state)->visible = true; - - dev_priv->display.update_primary_plane(crtc, plane->fb, - crtc->x, crtc->y); -} - static bool need_vtd_wa(struct drm_device *dev) { #ifdef CONFIG_INTEL_IOMMU @@ -2244,7 +2219,7 @@ static bool need_vtd_wa(struct drm_device *dev) unsigned int intel_tile_height(struct drm_device *dev, uint32_t pixel_format, - uint64_t fb_format_modifier) + uint64_t fb_format_modifier, unsigned int plane) { unsigned int tile_height; uint32_t pixel_bytes; @@ -2260,7 +2235,7 @@ intel_tile_height(struct drm_device *dev, uint32_t pixel_format, tile_height = 32; break; case I915_FORMAT_MOD_Yf_TILED: - pixel_bytes = drm_format_plane_cpp(pixel_format, 0); + pixel_bytes = drm_format_plane_cpp(pixel_format, plane); switch (pixel_bytes) { default: case 1: @@ -2294,38 +2269,68 @@ intel_fb_align_height(struct drm_device *dev, unsigned int height, uint32_t pixel_format, uint64_t fb_format_modifier) { return ALIGN(height, intel_tile_height(dev, pixel_format, - fb_format_modifier)); + fb_format_modifier, 0)); } -static int +static void intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, struct drm_framebuffer *fb, const struct drm_plane_state *plane_state) { - struct intel_rotation_info *info = &view->rotation_info; + struct intel_rotation_info *info = &view->params.rotation_info; + unsigned int tile_height, tile_pitch; *view = i915_ggtt_view_normal; if (!plane_state) - return 0; + return; if (!intel_rotation_90_or_270(plane_state->rotation)) - return 0; + return; *view = i915_ggtt_view_rotated; info->height = fb->height; info->pixel_format = fb->pixel_format; info->pitch = fb->pitches[0]; + info->uv_offset = fb->offsets[1]; info->fb_modifier = fb->modifier[0]; - return 0; + tile_height = intel_tile_height(fb->dev, fb->pixel_format, + fb->modifier[0], 0); + tile_pitch = PAGE_SIZE / tile_height; + info->width_pages = DIV_ROUND_UP(fb->pitches[0], tile_pitch); + info->height_pages = DIV_ROUND_UP(fb->height, tile_height); + info->size = info->width_pages * info->height_pages * PAGE_SIZE; + + if (info->pixel_format == DRM_FORMAT_NV12) { + tile_height = intel_tile_height(fb->dev, fb->pixel_format, + fb->modifier[0], 1); + tile_pitch = PAGE_SIZE / tile_height; + info->width_pages_uv = DIV_ROUND_UP(fb->pitches[0], tile_pitch); + info->height_pages_uv = DIV_ROUND_UP(fb->height / 2, + tile_height); + info->size_uv = info->width_pages_uv * info->height_pages_uv * + PAGE_SIZE; + } +} + +static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv) +{ + if (INTEL_INFO(dev_priv)->gen >= 9) + return 256 * 1024; + else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) || + IS_VALLEYVIEW(dev_priv)) + return 128 * 1024; + else if (INTEL_INFO(dev_priv)->gen >= 4) + return 4 * 1024; + else + return 0; } int intel_pin_and_fence_fb_obj(struct drm_plane *plane, struct drm_framebuffer *fb, - const struct drm_plane_state *plane_state, - struct intel_engine_cs *pipelined) + const struct drm_plane_state *plane_state) { struct drm_device *dev = fb->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -2338,14 +2343,7 @@ intel_pin_and_fence_fb_obj(struct drm_plane *plane, switch (fb->modifier[0]) { case DRM_FORMAT_MOD_NONE: - if (INTEL_INFO(dev)->gen >= 9) - alignment = 256 * 1024; - else if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) - alignment = 128 * 1024; - else if (INTEL_INFO(dev)->gen >= 4) - alignment = 4 * 1024; - else - alignment = 64 * 1024; + alignment = intel_linear_alignment(dev_priv); break; case I915_FORMAT_MOD_X_TILED: if (INTEL_INFO(dev)->gen >= 9) @@ -2367,9 +2365,7 @@ intel_pin_and_fence_fb_obj(struct drm_plane *plane, return -EINVAL; } - ret = intel_fill_fb_ggtt_view(&view, fb, plane_state); - if (ret) - return ret; + intel_fill_fb_ggtt_view(&view, fb, plane_state); /* Note that the w/a also requires 64 PTE of padding following the * bo. We currently fill all unused PTE with the shadow page and so @@ -2388,31 +2384,41 @@ intel_pin_and_fence_fb_obj(struct drm_plane *plane, */ intel_runtime_pm_get(dev_priv); - dev_priv->mm.interruptible = false; - ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined, + ret = i915_gem_object_pin_to_display_plane(obj, alignment, &view); if (ret) - goto err_interruptible; + goto err_pm; /* Install a fence for tiled scan-out. Pre-i965 always needs a * fence, whereas 965+ only requires a fence if using * framebuffer compression. For simplicity, we always install * a fence as the cost is not that onerous. */ - ret = i915_gem_object_get_fence(obj); - if (ret) - goto err_unpin; + if (view.type == I915_GGTT_VIEW_NORMAL) { + ret = i915_gem_object_get_fence(obj); + if (ret == -EDEADLK) { + /* + * -EDEADLK means there are no free fences + * no pending flips. + * + * This is propagated to atomic, but it uses + * -EDEADLK to force a locking recovery, so + * change the returned error to -EBUSY. + */ + ret = -EBUSY; + goto err_unpin; + } else if (ret) + goto err_unpin; - i915_gem_object_pin_fence(obj); + i915_gem_object_pin_fence(obj); + } - dev_priv->mm.interruptible = true; intel_runtime_pm_put(dev_priv); return 0; err_unpin: i915_gem_object_unpin_from_display_plane(obj, &view); -err_interruptible: - dev_priv->mm.interruptible = true; +err_pm: intel_runtime_pm_put(dev_priv); return ret; } @@ -2422,20 +2428,21 @@ static void intel_unpin_fb_obj(struct drm_framebuffer *fb, { struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct i915_ggtt_view view; - int ret; WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex)); - ret = intel_fill_fb_ggtt_view(&view, fb, plane_state); - WARN_ONCE(ret, "Couldn't get view from plane state!"); + intel_fill_fb_ggtt_view(&view, fb, plane_state); + + if (view.type == I915_GGTT_VIEW_NORMAL) + i915_gem_object_unpin_fence(obj); - i915_gem_object_unpin_fence(obj); i915_gem_object_unpin_from_display_plane(obj, &view); } /* Computes the linear offset to the base tile and adjusts x, y. bytes per pixel * is assumed to be a power-of-two. */ -unsigned long intel_gen4_compute_page_offset(int *x, int *y, +unsigned long intel_gen4_compute_page_offset(struct drm_i915_private *dev_priv, + int *x, int *y, unsigned int tiling_mode, unsigned int cpp, unsigned int pitch) @@ -2451,12 +2458,13 @@ unsigned long intel_gen4_compute_page_offset(int *x, int *y, return tile_rows * pitch * 8 + tiles * 4096; } else { + unsigned int alignment = intel_linear_alignment(dev_priv) - 1; unsigned int offset; offset = *y * pitch + *x * cpp; - *y = 0; - *x = (offset & 4095) / cpp; - return offset & -4096; + *y = (offset & alignment) / pitch; + *x = ((offset & alignment) - *y * pitch) / cpp; + return offset & ~alignment; } } @@ -2512,6 +2520,7 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc, struct intel_initial_plane_config *plane_config) { struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj = NULL; struct drm_mode_fb_cmd2 mode_cmd = { 0 }; struct drm_framebuffer *fb = &plane_config->fb->base; @@ -2524,6 +2533,12 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc, if (plane_config->size == 0) return false; + /* If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. */ + if (size_aligned * 2 > dev_priv->gtt.stolen_usable_size) + return false; + obj = i915_gem_object_create_stolen_for_preallocated(dev, base_aligned, base_aligned, @@ -2583,6 +2598,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, struct intel_crtc *i; struct drm_i915_gem_object *obj; struct drm_plane *primary = intel_crtc->base.primary; + struct drm_plane_state *plane_state = primary->state; struct drm_framebuffer *fb; if (!plane_config->fb) @@ -2622,15 +2638,25 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, return; valid_fb: + plane_state->src_x = 0; + plane_state->src_y = 0; + plane_state->src_w = fb->width << 16; + plane_state->src_h = fb->height << 16; + + plane_state->crtc_x = 0; + plane_state->crtc_y = 0; + plane_state->crtc_w = fb->width; + plane_state->crtc_h = fb->height; + obj = intel_fb_obj(fb); if (obj->tiling_mode != I915_TILING_NONE) dev_priv->preserve_bios_swizzle = true; - primary->fb = fb; - primary->state->crtc = &intel_crtc->base; - primary->crtc = &intel_crtc->base; - update_state_fb(primary); - obj->frontbuffer_bits |= INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe); + drm_framebuffer_reference(fb); + primary->fb = primary->state->fb = fb; + primary->crtc = primary->state->crtc = &intel_crtc->base; + intel_crtc->base.state->plane_mask |= (1 << drm_plane_index(primary)); + obj->frontbuffer_bits |= to_intel_plane(primary)->frontbuffer_bit; } static void i9xx_update_primary_plane(struct drm_crtc *crtc, @@ -2646,7 +2672,7 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc, int plane = intel_crtc->plane; unsigned long linear_offset; u32 dspcntr; - u32 reg = DSPCNTR(plane); + i915_reg_t reg = DSPCNTR(plane); int pixel_size; if (!visible || !fb) { @@ -2725,7 +2751,8 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc, if (INTEL_INFO(dev)->gen >= 4) { intel_crtc->dspaddr_offset = - intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, + intel_gen4_compute_page_offset(dev_priv, + &x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); linear_offset -= intel_crtc->dspaddr_offset; @@ -2746,6 +2773,9 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc, (intel_crtc->config->pipe_src_w - 1) * pixel_size; } + intel_crtc->adjusted_x = x; + intel_crtc->adjusted_y = y; + I915_WRITE(reg, dspcntr); I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]); @@ -2772,7 +2802,7 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc, int plane = intel_crtc->plane; unsigned long linear_offset; u32 dspcntr; - u32 reg = DSPCNTR(plane); + i915_reg_t reg = DSPCNTR(plane); int pixel_size; if (!visible || !fb) { @@ -2826,7 +2856,8 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc, linear_offset = y * fb->pitches[0] + x * pixel_size; intel_crtc->dspaddr_offset = - intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, + intel_gen4_compute_page_offset(dev_priv, + &x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); linear_offset -= intel_crtc->dspaddr_offset; @@ -2845,6 +2876,9 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc, } } + intel_crtc->adjusted_x = x; + intel_crtc->adjusted_y = y; + I915_WRITE(reg, dspcntr); I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]); @@ -2893,43 +2927,58 @@ u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier, } } -unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane, - struct drm_i915_gem_object *obj) +u32 intel_plane_obj_offset(struct intel_plane *intel_plane, + struct drm_i915_gem_object *obj, + unsigned int plane) { - const struct i915_ggtt_view *view = &i915_ggtt_view_normal; + struct i915_ggtt_view view; + struct i915_vma *vma; + u64 offset; + + intel_fill_fb_ggtt_view(&view, intel_plane->base.fb, + intel_plane->base.state); + + vma = i915_gem_obj_to_ggtt_view(obj, &view); + if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n", + view.type)) + return -1; + + offset = vma->node.start; + + if (plane == 1) { + offset += vma->ggtt_view.params.rotation_info.uv_start_page * + PAGE_SIZE; + } + + WARN_ON(upper_32_bits(offset)); - if (intel_rotation_90_or_270(intel_plane->base.state->rotation)) - view = &i915_ggtt_view_rotated; + return lower_32_bits(offset); +} + +static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id) +{ + struct drm_device *dev = intel_crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; - return i915_gem_obj_ggtt_offset_view(obj, view); + I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0); + I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0); + I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0); } /* * This function detaches (aka. unbinds) unused scalers in hardware */ -void skl_detach_scalers(struct intel_crtc *intel_crtc) +static void skl_detach_scalers(struct intel_crtc *intel_crtc) { - struct drm_device *dev; - struct drm_i915_private *dev_priv; struct intel_crtc_scaler_state *scaler_state; int i; - if (!intel_crtc || !intel_crtc->config) - return; - - dev = intel_crtc->base.dev; - dev_priv = dev->dev_private; scaler_state = &intel_crtc->config->scaler_state; /* loop through and disable scalers that aren't in use */ for (i = 0; i < intel_crtc->num_scalers; i++) { - if (!scaler_state->scalers[i].in_use) { - I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, i), 0); - I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, i), 0); - I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, i), 0); - DRM_DEBUG_KMS("CRTC:%d Disabled scaler id %u.%u\n", - intel_crtc->base.base.id, intel_crtc->pipe, i); - } + if (!scaler_state->scalers[i].in_use) + skl_detach_scaler(intel_crtc, i); } } @@ -3029,7 +3078,7 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc, u32 tile_height, plane_offset, plane_size; unsigned int rotation; int x_offset, y_offset; - unsigned long surf_addr; + u32 surf_addr; struct intel_crtc_state *crtc_state = intel_crtc->config; struct intel_plane_state *plane_state; int src_x = 0, src_y = 0, src_w = 0, src_h = 0; @@ -3059,34 +3108,26 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc, obj = intel_fb_obj(fb); stride_div = intel_fb_stride_alignment(dev, fb->modifier[0], fb->pixel_format); - surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj); + surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0); - /* - * FIXME: intel_plane_state->src, dst aren't set when transitional - * update_plane helpers are called from legacy paths. - * Once full atomic crtc is available, below check can be avoided. - */ - if (drm_rect_width(&plane_state->src)) { - scaler_id = plane_state->scaler_id; - src_x = plane_state->src.x1 >> 16; - src_y = plane_state->src.y1 >> 16; - src_w = drm_rect_width(&plane_state->src) >> 16; - src_h = drm_rect_height(&plane_state->src) >> 16; - dst_x = plane_state->dst.x1; - dst_y = plane_state->dst.y1; - dst_w = drm_rect_width(&plane_state->dst); - dst_h = drm_rect_height(&plane_state->dst); - - WARN_ON(x != src_x || y != src_y); - } else { - src_w = intel_crtc->config->pipe_src_w; - src_h = intel_crtc->config->pipe_src_h; - } + WARN_ON(drm_rect_width(&plane_state->src) == 0); + + scaler_id = plane_state->scaler_id; + src_x = plane_state->src.x1 >> 16; + src_y = plane_state->src.y1 >> 16; + src_w = drm_rect_width(&plane_state->src) >> 16; + src_h = drm_rect_height(&plane_state->src) >> 16; + dst_x = plane_state->dst.x1; + dst_y = plane_state->dst.y1; + dst_w = drm_rect_width(&plane_state->dst); + dst_h = drm_rect_height(&plane_state->dst); + + WARN_ON(x != src_x || y != src_y); if (intel_rotation_90_or_270(rotation)) { /* stride = Surface height in tiles */ tile_height = intel_tile_height(dev, fb->pixel_format, - fb->modifier[0]); + fb->modifier[0], 0); stride = DIV_ROUND_UP(fb->height, tile_height); x_offset = stride * tile_height - y - src_h; y_offset = x; @@ -3099,6 +3140,9 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc, } plane_offset = y_offset << 16 | x_offset; + intel_crtc->adjusted_x = x_offset; + intel_crtc->adjusted_y = y_offset; + I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl); I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset); I915_WRITE(PLANE_SIZE(pipe, 0), plane_size); @@ -3132,8 +3176,8 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->display.disable_fbc) - dev_priv->display.disable_fbc(dev); + if (dev_priv->fbc.deactivate) + dev_priv->fbc.deactivate(dev_priv); dev_priv->display.update_primary_plane(crtc, fb, x, y); @@ -3155,45 +3199,24 @@ static void intel_complete_page_flips(struct drm_device *dev) static void intel_update_primary_planes(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; for_each_crtc(dev, crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - - drm_modeset_lock(&crtc->mutex, NULL); - /* - * FIXME: Once we have proper support for primary planes (and - * disabling them without disabling the entire crtc) allow again - * a NULL crtc->primary->fb. - */ - if (intel_crtc->active && crtc->primary->fb) - dev_priv->display.update_primary_plane(crtc, - crtc->primary->fb, - crtc->x, - crtc->y); - drm_modeset_unlock(&crtc->mutex); - } -} + struct intel_plane *plane = to_intel_plane(crtc->primary); + struct intel_plane_state *plane_state; -void intel_crtc_reset(struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + drm_modeset_lock_crtc(crtc, &plane->base); + plane_state = to_intel_plane_state(plane->base.state); - if (!crtc->active) - return; + if (crtc->state->active && plane_state->base.fb) + plane->commit_plane(&plane->base, plane_state); - intel_crtc_disable_planes(&crtc->base); - dev_priv->display.crtc_disable(&crtc->base); - dev_priv->display.crtc_enable(&crtc->base); - intel_crtc_enable_planes(&crtc->base); + drm_modeset_unlock_crtc(crtc); + } } void intel_prepare_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *crtc; - /* no reset support for gen2 */ if (IS_GEN2(dev)) return; @@ -3203,18 +3226,11 @@ void intel_prepare_reset(struct drm_device *dev) return; drm_modeset_lock_all(dev); - /* * Disabling the crtcs gracefully seems nicer. Also the * g33 docs say we should at least disable all the planes. */ - for_each_intel_crtc(dev, crtc) { - if (!crtc->active) - continue; - - intel_crtc_disable_planes(&crtc->base); - dev_priv->display.crtc_disable(&crtc->base); - } + intel_display_suspend(dev); } void intel_finish_reset(struct drm_device *dev) @@ -3239,6 +3255,9 @@ void intel_finish_reset(struct drm_device *dev) * so update the base address of all primary * planes to the the last fb to make sure we're * showing the correct fb after a reset. + * + * FIXME: Atomic will make this obsolete since we won't schedule + * CS-based flips (which might get lost in gpu resets) any more. */ intel_update_primary_planes(dev); return; @@ -3258,39 +3277,13 @@ void intel_finish_reset(struct drm_device *dev) dev_priv->display.hpd_irq_setup(dev); spin_unlock_irq(&dev_priv->irq_lock); - intel_modeset_setup_hw_state(dev, true); + intel_display_resume(dev); intel_hpd_init(dev_priv); drm_modeset_unlock_all(dev); } -static void -intel_finish_fb(struct drm_framebuffer *old_fb) -{ - struct drm_i915_gem_object *obj = intel_fb_obj(old_fb); - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - bool was_interruptible = dev_priv->mm.interruptible; - int ret; - - /* Big Hammer, we also need to ensure that any pending - * MI_WAIT_FOR_EVENT inside a user batch buffer on the - * current scanout is retired before unpinning the old - * framebuffer. Note that we rely on userspace rendering - * into the buffer attached to the pipe they are waiting - * on. If not, userspace generates a GPU hang with IPEHR - * point to the MI_WAIT_FOR_EVENT. - * - * This should only fail upon a hung GPU, in which case we - * can safely continue. - */ - dev_priv->mm.interruptible = false; - ret = i915_gem_object_wait_rendering(obj, true); - dev_priv->mm.interruptible = was_interruptible; - - WARN_ON(ret); -} - static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; @@ -3309,14 +3302,23 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) return pending; } -static void intel_update_pipe_size(struct intel_crtc *crtc) +static void intel_update_pipe_config(struct intel_crtc *crtc, + struct intel_crtc_state *old_crtc_state) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - const struct drm_display_mode *adjusted_mode; + struct intel_crtc_state *pipe_config = + to_intel_crtc_state(crtc->base.state); - if (!i915.fastboot) - return; + /* drm_atomic_helper_update_legacy_modeset_state might not be called. */ + crtc->base.mode = crtc->base.state->mode; + + DRM_DEBUG_KMS("Updating pipe size %ix%i -> %ix%i\n", + old_crtc_state->pipe_src_w, old_crtc_state->pipe_src_h, + pipe_config->pipe_src_w, pipe_config->pipe_src_h); + + if (HAS_DDI(dev)) + intel_set_pipe_csc(&crtc->base); /* * Update pipe size and adjust fitter if needed: the reason for this is @@ -3325,27 +3327,24 @@ static void intel_update_pipe_size(struct intel_crtc *crtc) * fastboot case, we'll flip, but if we don't update the pipesrc and * pfit state, we'll end up with a big fb scanned out into the wrong * sized surface. - * - * To fix this properly, we need to hoist the checks up into - * compute_mode_changes (or above), check the actual pfit state and - * whether the platform allows pfit disable with pipe active, and only - * then update the pipesrc and pfit state, even on the flip path. */ - adjusted_mode = &crtc->config->base.adjusted_mode; - I915_WRITE(PIPESRC(crtc->pipe), - ((adjusted_mode->crtc_hdisplay - 1) << 16) | - (adjusted_mode->crtc_vdisplay - 1)); - if (!crtc->config->pch_pfit.enabled && - (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) || - intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) { - I915_WRITE(PF_CTL(crtc->pipe), 0); - I915_WRITE(PF_WIN_POS(crtc->pipe), 0); - I915_WRITE(PF_WIN_SZ(crtc->pipe), 0); + ((pipe_config->pipe_src_w - 1) << 16) | + (pipe_config->pipe_src_h - 1)); + + /* on skylake this is done by detaching scalers */ + if (INTEL_INFO(dev)->gen >= 9) { + skl_detach_scalers(crtc); + + if (pipe_config->pch_pfit.enabled) + skylake_pfit_enable(crtc); + } else if (HAS_PCH_SPLIT(dev)) { + if (pipe_config->pch_pfit.enabled) + ironlake_pfit_enable(crtc); + else if (old_crtc_state->pch_pfit.enabled) + ironlake_pfit_disable(crtc, true); } - crtc->config->pipe_src_w = adjusted_mode->crtc_hdisplay; - crtc->config->pipe_src_h = adjusted_mode->crtc_vdisplay; } static void intel_fdi_normal_train(struct drm_crtc *crtc) @@ -3354,7 +3353,8 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - u32 reg, temp; + i915_reg_t reg; + u32 temp; /* enable normal train */ reg = FDI_TX_CTL(pipe); @@ -3396,7 +3396,8 @@ static void ironlake_fdi_link_train(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - u32 reg, temp, tries; + i915_reg_t reg; + u32 temp, tries; /* FDI needs bits from pipe first */ assert_pipe_enabled(dev_priv, pipe); @@ -3496,7 +3497,8 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - u32 reg, temp, i, retry; + i915_reg_t reg; + u32 temp, i, retry; /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ @@ -3628,7 +3630,8 @@ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - u32 reg, temp, i, j; + i915_reg_t reg; + u32 temp, i, j; /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ @@ -3745,8 +3748,8 @@ static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc) struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int pipe = intel_crtc->pipe; - u32 reg, temp; - + i915_reg_t reg; + u32 temp; /* enable PCH FDI RX PLL, wait warmup plus DMI latency */ reg = FDI_RX_CTL(pipe); @@ -3782,7 +3785,8 @@ static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc) struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int pipe = intel_crtc->pipe; - u32 reg, temp; + i915_reg_t reg; + u32 temp; /* Switch from PCDclk to Rawclk */ reg = FDI_RX_CTL(pipe); @@ -3812,7 +3816,8 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - u32 reg, temp; + i915_reg_t reg; + u32 temp; /* disable CPU FDI tx and PCH FDI rx */ reg = FDI_TX_CTL(pipe); @@ -3905,15 +3910,23 @@ static void page_flip_completed(struct intel_crtc *intel_crtc) work->pending_flip_obj); } -void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) +static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; + long ret; WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue)); - if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue, - !intel_crtc_has_pending_flip(crtc), - 60*HZ) == 0)) { + + ret = wait_event_interruptible_timeout( + dev_priv->pending_flip_queue, + !intel_crtc_has_pending_flip(crtc), + 60*HZ); + + if (ret < 0) + return ret; + + if (ret == 0) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); spin_lock_irq(&dev->event_lock); @@ -3924,11 +3937,7 @@ void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) spin_unlock_irq(&dev->event_lock); } - if (crtc->primary->fb) { - mutex_lock(&dev->struct_mutex); - intel_finish_fb(crtc->primary->fb); - mutex_unlock(&dev->struct_mutex); - } + return 0; } /* Program iCLKIP clock to the desired frequency */ @@ -4088,6 +4097,22 @@ static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc) } } +/* Return which DP Port should be selected for Transcoder DP control */ +static enum port +intel_trans_dp_port_sel(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct intel_encoder *encoder; + + for_each_encoder_on_crtc(dev, crtc, encoder) { + if (encoder->type == INTEL_OUTPUT_DISPLAYPORT || + encoder->type == INTEL_OUTPUT_EDP) + return enc_to_dig_port(&encoder->base)->port; + } + + return -1; +} + /* * Enable PCH resources required for PCH ports: * - PCH PLLs @@ -4102,7 +4127,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - u32 reg, temp; + u32 temp; assert_pch_transcoder_disabled(dev_priv, pipe); @@ -4114,6 +4139,12 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) I915_WRITE(FDI_RX_TUSIZE1(pipe), I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK); + /* + * Sometimes spurious CPU pipe underruns happen during FDI + * training, at least with VGA+HDMI cloning. Suppress them. + */ + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); + /* For PCH output, training FDI link */ dev_priv->display.fdi_link_train(crtc); @@ -4147,10 +4178,14 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) intel_fdi_normal_train(crtc); + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); + /* For PCH DP, enable TRANS_DP_CTL */ if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) { + const struct drm_display_mode *adjusted_mode = + &intel_crtc->config->base.adjusted_mode; u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5; - reg = TRANS_DP_CTL(pipe); + i915_reg_t reg = TRANS_DP_CTL(pipe); temp = I915_READ(reg); temp &= ~(TRANS_DP_PORT_SEL_MASK | TRANS_DP_SYNC_MASK | @@ -4158,19 +4193,19 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) temp |= TRANS_DP_OUTPUT_ENABLE; temp |= bpc << 9; /* same format but at 11:9 */ - if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC) + if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) temp |= TRANS_DP_HSYNC_ACTIVE_HIGH; - if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC) + if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) temp |= TRANS_DP_VSYNC_ACTIVE_HIGH; switch (intel_trans_dp_port_sel(crtc)) { - case PCH_DP_B: + case PORT_B: temp |= TRANS_DP_PORT_SEL_B; break; - case PCH_DP_C: + case PORT_C: temp |= TRANS_DP_PORT_SEL_C; break; - case PCH_DP_D: + case PORT_D: temp |= TRANS_DP_PORT_SEL_D; break; default: @@ -4200,33 +4235,16 @@ static void lpt_pch_enable(struct drm_crtc *crtc) lpt_enable_pch_transcoder(dev_priv, cpu_transcoder); } -void intel_put_shared_dpll(struct intel_crtc *crtc) -{ - struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc); - - if (pll == NULL) - return; - - if (!(pll->config.crtc_mask & (1 << crtc->pipe))) { - WARN(1, "bad %s crtc mask\n", pll->name); - return; - } - - pll->config.crtc_mask &= ~(1 << crtc->pipe); - if (pll->config.crtc_mask == 0) { - WARN_ON(pll->on); - WARN_ON(pll->active); - } - - crtc->config->shared_dpll = DPLL_ID_PRIVATE; -} - struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; struct intel_shared_dpll *pll; + struct intel_shared_dpll_config *shared_dpll; enum intel_dpll_id i; + int max = dev_priv->num_shared_dpll; + + shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state); if (HAS_PCH_IBX(dev_priv->dev)) { /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */ @@ -4236,7 +4254,7 @@ struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n", crtc->base.base.id, pll->name); - WARN_ON(pll->new_config->crtc_mask); + WARN_ON(shared_dpll[i].crtc_mask); goto found; } @@ -4256,24 +4274,26 @@ struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, pll = &dev_priv->shared_dplls[i]; DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n", crtc->base.base.id, pll->name); - WARN_ON(pll->new_config->crtc_mask); + WARN_ON(shared_dpll[i].crtc_mask); goto found; - } + } else if (INTEL_INFO(dev_priv)->gen < 9 && HAS_DDI(dev_priv)) + /* Do not consider SPLL */ + max = 2; - for (i = 0; i < dev_priv->num_shared_dpll; i++) { + for (i = 0; i < max; i++) { pll = &dev_priv->shared_dplls[i]; /* Only want to check enabled timings first */ - if (pll->new_config->crtc_mask == 0) + if (shared_dpll[i].crtc_mask == 0) continue; if (memcmp(&crtc_state->dpll_hw_state, - &pll->new_config->hw_state, - sizeof(pll->new_config->hw_state)) == 0) { + &shared_dpll[i].hw_state, + sizeof(crtc_state->dpll_hw_state)) == 0) { DRM_DEBUG_KMS("CRTC:%d sharing existing %s (crtc mask 0x%08x, ative %d)\n", crtc->base.base.id, pll->name, - pll->new_config->crtc_mask, + shared_dpll[i].crtc_mask, pll->active); goto found; } @@ -4282,7 +4302,7 @@ struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, /* Ok no matching timings, maybe there's a free one? */ for (i = 0; i < dev_priv->num_shared_dpll; i++) { pll = &dev_priv->shared_dplls[i]; - if (pll->new_config->crtc_mask == 0) { + if (shared_dpll[i].crtc_mask == 0) { DRM_DEBUG_KMS("CRTC:%d allocated %s\n", crtc->base.base.id, pll->name); goto found; @@ -4292,90 +4312,40 @@ struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, return NULL; found: - if (pll->new_config->crtc_mask == 0) - pll->new_config->hw_state = crtc_state->dpll_hw_state; + if (shared_dpll[i].crtc_mask == 0) + shared_dpll[i].hw_state = + crtc_state->dpll_hw_state; crtc_state->shared_dpll = i; DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name, pipe_name(crtc->pipe)); - pll->new_config->crtc_mask |= 1 << crtc->pipe; + shared_dpll[i].crtc_mask |= 1 << crtc->pipe; return pll; } -/** - * intel_shared_dpll_start_config - start a new PLL staged config - * @dev_priv: DRM device - * @clear_pipes: mask of pipes that will have their PLLs freed - * - * Starts a new PLL staged config, copying the current config but - * releasing the references of pipes specified in clear_pipes. - */ -static int intel_shared_dpll_start_config(struct drm_i915_private *dev_priv, - unsigned clear_pipes) -{ - struct intel_shared_dpll *pll; - enum intel_dpll_id i; - - for (i = 0; i < dev_priv->num_shared_dpll; i++) { - pll = &dev_priv->shared_dplls[i]; - - pll->new_config = kmemdup(&pll->config, sizeof pll->config, - GFP_KERNEL); - if (!pll->new_config) - goto cleanup; - - pll->new_config->crtc_mask &= ~clear_pipes; - } - - return 0; - -cleanup: - while (--i >= 0) { - pll = &dev_priv->shared_dplls[i]; - kfree(pll->new_config); - pll->new_config = NULL; - } - - return -ENOMEM; -} - -static void intel_shared_dpll_commit(struct drm_i915_private *dev_priv) +static void intel_shared_dpll_commit(struct drm_atomic_state *state) { + struct drm_i915_private *dev_priv = to_i915(state->dev); + struct intel_shared_dpll_config *shared_dpll; struct intel_shared_dpll *pll; enum intel_dpll_id i; - for (i = 0; i < dev_priv->num_shared_dpll; i++) { - pll = &dev_priv->shared_dplls[i]; - - WARN_ON(pll->new_config == &pll->config); - - pll->config = *pll->new_config; - kfree(pll->new_config); - pll->new_config = NULL; - } -} - -static void intel_shared_dpll_abort_config(struct drm_i915_private *dev_priv) -{ - struct intel_shared_dpll *pll; - enum intel_dpll_id i; + if (!to_intel_atomic_state(state)->dpll_set) + return; + shared_dpll = to_intel_atomic_state(state)->shared_dpll; for (i = 0; i < dev_priv->num_shared_dpll; i++) { pll = &dev_priv->shared_dplls[i]; - - WARN_ON(pll->new_config == &pll->config); - - kfree(pll->new_config); - pll->new_config = NULL; + pll->config = shared_dpll[i]; } } static void cpt_verify_modeset(struct drm_device *dev, int pipe) { struct drm_i915_private *dev_priv = dev->dev_private; - int dslreg = PIPEDSL(pipe); + i915_reg_t dslreg = PIPEDSL(pipe); u32 temp; temp = I915_READ(dslreg); @@ -4386,62 +4356,16 @@ static void cpt_verify_modeset(struct drm_device *dev, int pipe) } } -/** - * skl_update_scaler_users - Stages update to crtc's scaler state - * @intel_crtc: crtc - * @crtc_state: crtc_state - * @plane: plane (NULL indicates crtc is requesting update) - * @plane_state: plane's state - * @force_detach: request unconditional detachment of scaler - * - * This function updates scaler state for requested plane or crtc. - * To request scaler usage update for a plane, caller shall pass plane pointer. - * To request scaler usage update for crtc, caller shall pass plane pointer - * as NULL. - * - * Return - * 0 - scaler_usage updated successfully - * error - requested scaling cannot be supported or other error condition - */ -int -skl_update_scaler_users( - struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state, - struct intel_plane *intel_plane, struct intel_plane_state *plane_state, - int force_detach) +static int +skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, + unsigned scaler_user, int *scaler_id, unsigned int rotation, + int src_w, int src_h, int dst_w, int dst_h) { + struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; + struct intel_crtc *intel_crtc = + to_intel_crtc(crtc_state->base.crtc); int need_scaling; - int idx; - int src_w, src_h, dst_w, dst_h; - int *scaler_id; - struct drm_framebuffer *fb; - struct intel_crtc_scaler_state *scaler_state; - unsigned int rotation; - - if (!intel_crtc || !crtc_state) - return 0; - - scaler_state = &crtc_state->scaler_state; - - idx = intel_plane ? drm_plane_index(&intel_plane->base) : SKL_CRTC_INDEX; - fb = intel_plane ? plane_state->base.fb : NULL; - - if (intel_plane) { - src_w = drm_rect_width(&plane_state->src) >> 16; - src_h = drm_rect_height(&plane_state->src) >> 16; - dst_w = drm_rect_width(&plane_state->dst); - dst_h = drm_rect_height(&plane_state->dst); - scaler_id = &plane_state->scaler_id; - rotation = plane_state->base.rotation; - } else { - struct drm_display_mode *adjusted_mode = - &crtc_state->base.adjusted_mode; - src_w = crtc_state->pipe_src_w; - src_h = crtc_state->pipe_src_h; - dst_w = adjusted_mode->hdisplay; - dst_h = adjusted_mode->vdisplay; - scaler_id = &scaler_state->scaler_id; - rotation = DRM_ROTATE_0; - } need_scaling = intel_rotation_90_or_270(rotation) ? (src_h != dst_w || src_w != dst_h): @@ -4457,17 +4381,14 @@ skl_update_scaler_users( * update to free the scaler is done in plane/panel-fit programming. * For this purpose crtc/plane_state->scaler_id isn't reset here. */ - if (force_detach || !need_scaling || (intel_plane && - (!fb || !plane_state->visible))) { + if (force_detach || !need_scaling) { if (*scaler_id >= 0) { - scaler_state->scaler_users &= ~(1 << idx); + scaler_state->scaler_users &= ~(1 << scaler_user); scaler_state->scalers[*scaler_id].in_use = 0; - DRM_DEBUG_KMS("Staged freeing scaler id %d.%d from %s:%d " - "crtc_state = %p scaler_users = 0x%x\n", - intel_crtc->pipe, *scaler_id, intel_plane ? "PLANE" : "CRTC", - intel_plane ? intel_plane->base.base.id : - intel_crtc->base.base.id, crtc_state, + DRM_DEBUG_KMS("scaler_user index %u.%u: " + "Staged freeing scaler id %d scaler_users = 0x%x\n", + intel_crtc->pipe, scaler_user, *scaler_id, scaler_state->scaler_users); *scaler_id = -1; } @@ -4480,55 +4401,122 @@ skl_update_scaler_users( src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H || dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H) { - DRM_DEBUG_KMS("%s:%d scaler_user index %u.%u: src %ux%u dst %ux%u " + DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u " "size is out of scaler range\n", - intel_plane ? "PLANE" : "CRTC", - intel_plane ? intel_plane->base.base.id : intel_crtc->base.base.id, - intel_crtc->pipe, idx, src_w, src_h, dst_w, dst_h); + intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h); return -EINVAL; } + /* mark this plane as a scaler user in crtc_state */ + scaler_state->scaler_users |= (1 << scaler_user); + DRM_DEBUG_KMS("scaler_user index %u.%u: " + "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n", + intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h, + scaler_state->scaler_users); + + return 0; +} + +/** + * skl_update_scaler_crtc - Stages update to scaler state for a given crtc. + * + * @state: crtc's scaler state + * + * Return + * 0 - scaler_usage updated successfully + * error - requested scaling cannot be supported or other error condition + */ +int skl_update_scaler_crtc(struct intel_crtc_state *state) +{ + struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc); + const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode; + + DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n", + intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX); + + return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, + &state->scaler_state.scaler_id, DRM_ROTATE_0, + state->pipe_src_w, state->pipe_src_h, + adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay); +} + +/** + * skl_update_scaler_plane - Stages update to scaler state for a given plane. + * + * @state: crtc's scaler state + * @plane_state: atomic plane state to update + * + * Return + * 0 - scaler_usage updated successfully + * error - requested scaling cannot be supported or other error condition + */ +static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) +{ + + struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); + struct intel_plane *intel_plane = + to_intel_plane(plane_state->base.plane); + struct drm_framebuffer *fb = plane_state->base.fb; + int ret; + + bool force_detach = !fb || !plane_state->visible; + + DRM_DEBUG_KMS("Updating scaler for [PLANE:%d] scaler_user index %u.%u\n", + intel_plane->base.base.id, intel_crtc->pipe, + drm_plane_index(&intel_plane->base)); + + ret = skl_update_scaler(crtc_state, force_detach, + drm_plane_index(&intel_plane->base), + &plane_state->scaler_id, + plane_state->base.rotation, + drm_rect_width(&plane_state->src) >> 16, + drm_rect_height(&plane_state->src) >> 16, + drm_rect_width(&plane_state->dst), + drm_rect_height(&plane_state->dst)); + + if (ret || plane_state->scaler_id < 0) + return ret; + /* check colorkey */ - if (WARN_ON(intel_plane && - intel_plane->ckey.flags != I915_SET_COLORKEY_NONE)) { - DRM_DEBUG_KMS("PLANE:%d scaling %ux%u->%ux%u not allowed with colorkey", - intel_plane->base.base.id, src_w, src_h, dst_w, dst_h); + if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) { + DRM_DEBUG_KMS("[PLANE:%d] scaling with color key not allowed", + intel_plane->base.base.id); return -EINVAL; } /* Check src format */ - if (intel_plane) { - switch (fb->pixel_format) { - case DRM_FORMAT_RGB565: - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ABGR8888: - case DRM_FORMAT_ARGB8888: - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_XBGR2101010: - case DRM_FORMAT_YUYV: - case DRM_FORMAT_YVYU: - case DRM_FORMAT_UYVY: - case DRM_FORMAT_VYUY: - break; - default: - DRM_DEBUG_KMS("PLANE:%d FB:%d unsupported scaling format 0x%x\n", - intel_plane->base.base.id, fb->base.id, fb->pixel_format); - return -EINVAL; - } + switch (fb->pixel_format) { + case DRM_FORMAT_RGB565: + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ABGR8888: + case DRM_FORMAT_ARGB8888: + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_XBGR2101010: + case DRM_FORMAT_YUYV: + case DRM_FORMAT_YVYU: + case DRM_FORMAT_UYVY: + case DRM_FORMAT_VYUY: + break; + default: + DRM_DEBUG_KMS("[PLANE:%d] FB:%d unsupported scaling format 0x%x\n", + intel_plane->base.base.id, fb->base.id, fb->pixel_format); + return -EINVAL; } - /* mark this plane as a scaler user in crtc_state */ - scaler_state->scaler_users |= (1 << idx); - DRM_DEBUG_KMS("%s:%d staged scaling request for %ux%u->%ux%u " - "crtc_state = %p scaler_users = 0x%x\n", - intel_plane ? "PLANE" : "CRTC", - intel_plane ? intel_plane->base.base.id : intel_crtc->base.base.id, - src_w, src_h, dst_w, dst_h, crtc_state, scaler_state->scaler_users); return 0; } -static void skylake_pfit_update(struct intel_crtc *crtc, int enable) +static void skylake_scaler_disable(struct intel_crtc *crtc) +{ + int i; + + for (i = 0; i < crtc->num_scalers; i++) + skl_detach_scaler(crtc, i); +} + +static void skylake_pfit_enable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -4538,13 +4526,6 @@ static void skylake_pfit_update(struct intel_crtc *crtc, int enable) DRM_DEBUG_KMS("for crtc_state = %p\n", crtc->config); - /* To update pfit, first update scaler state */ - skl_update_scaler_users(crtc, crtc->config, NULL, NULL, !enable); - intel_atomic_setup_scalers(crtc->base.dev, crtc, crtc->config); - skl_detach_scalers(crtc); - if (!enable) - return; - if (crtc->config->pch_pfit.enabled) { int id; @@ -4584,20 +4565,6 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc) } } -static void intel_enable_sprite_planes(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - enum pipe pipe = to_intel_crtc(crtc)->pipe; - struct drm_plane *plane; - struct intel_plane *intel_plane; - - drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { - intel_plane = to_intel_plane(plane); - if (intel_plane->pipe == pipe) - intel_plane_restore(&intel_plane->base); - } -} - void hsw_enable_ips(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -4663,25 +4630,20 @@ static void intel_crtc_load_lut(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; - int palreg = PALETTE(pipe); int i; bool reenable_ips = false; /* The clocks have to be on to load the palette. */ - if (!crtc->state->enable || !intel_crtc->active) + if (!crtc->state->active) return; if (HAS_GMCH_DISPLAY(dev_priv->dev)) { - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) + if (intel_crtc->config->has_dsi_encoder) assert_dsi_pll_enabled(dev_priv); else assert_pll_enabled(dev_priv, pipe); } - /* use legacy palette for Ironlake */ - if (!HAS_GMCH_DISPLAY(dev)) - palreg = LGC_PALETTE(pipe); - /* Workaround : Do not read or write the pipe palette/gamma data while * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. */ @@ -4693,7 +4655,14 @@ static void intel_crtc_load_lut(struct drm_crtc *crtc) } for (i = 0; i < 256; i++) { - I915_WRITE(palreg + 4 * i, + i915_reg_t palreg; + + if (HAS_GMCH_DISPLAY(dev)) + palreg = PALETTE(pipe, i); + else + palreg = LGC_PALETTE(pipe, i); + + I915_WRITE(palreg, (intel_crtc->lut_r[i] << 16) | (intel_crtc->lut_g[i] << 8) | intel_crtc->lut_b[i]); @@ -4755,10 +4724,6 @@ intel_post_enable_primary(struct drm_crtc *crtc) */ hsw_enable_ips(intel_crtc); - mutex_lock(&dev->struct_mutex); - intel_fbc_update(dev); - mutex_unlock(&dev->struct_mutex); - /* * Gen2 reports pipe underruns whenever all planes are disabled. * So don't enable underrun reporting before at least some planes @@ -4769,9 +4734,9 @@ intel_post_enable_primary(struct drm_crtc *crtc) if (IS_GEN2(dev)) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); - /* Underruns don't raise interrupts, so check manually. */ - if (HAS_GMCH_DISPLAY(dev)) - i9xx_check_fifo_underruns(dev_priv); + /* Underruns don't always raise interrupts, so check manually. */ + intel_check_cpu_fifo_underruns(dev_priv); + intel_check_pch_fifo_underruns(dev_priv); } /** @@ -4810,13 +4775,11 @@ intel_pre_disable_primary(struct drm_crtc *crtc) * event which is after the vblank start event, so we need to have a * wait-for-vblank between disabling the plane and the pipe. */ - if (HAS_GMCH_DISPLAY(dev)) + if (HAS_GMCH_DISPLAY(dev)) { intel_set_memory_cxsr(dev_priv, false); - - mutex_lock(&dev->struct_mutex); - if (dev_priv->fbc.crtc == intel_crtc) - intel_fbc_disable(dev); - mutex_unlock(&dev->struct_mutex); + dev_priv->wm.vlv.cxsr = false; + intel_wait_for_vblank(dev, pipe); + } /* * FIXME IPS should be fine as long as one plane is @@ -4827,49 +4790,63 @@ intel_pre_disable_primary(struct drm_crtc *crtc) hsw_disable_ips(intel_crtc); } -static void intel_crtc_enable_planes(struct drm_crtc *crtc) +static void intel_post_plane_update(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; + struct intel_crtc_atomic_commit *atomic = &crtc->atomic; + struct drm_device *dev = crtc->base.dev; - intel_enable_primary_hw_plane(crtc->primary, crtc); - intel_enable_sprite_planes(crtc); - intel_crtc_update_cursor(crtc, true); + if (atomic->wait_vblank) + intel_wait_for_vblank(dev, crtc->pipe); - intel_post_enable_primary(crtc); + intel_frontbuffer_flip(dev, atomic->fb_bits); - /* - * FIXME: Once we grow proper nuclear flip support out of this we need - * to compute the mask of flip planes precisely. For the time being - * consider this a flip to a NULL plane. - */ - intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe)); + if (atomic->disable_cxsr) + crtc->wm.cxsr_allowed = true; + + if (crtc->atomic.update_wm_post) + intel_update_watermarks(&crtc->base); + + if (atomic->update_fbc) + intel_fbc_update(crtc); + + if (atomic->post_enable_primary) + intel_post_enable_primary(&crtc->base); + + memset(atomic, 0, sizeof(*atomic)); } -static void intel_crtc_disable_planes(struct drm_crtc *crtc) +static void intel_pre_plane_update(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_plane *intel_plane; - int pipe = intel_crtc->pipe; + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc_atomic_commit *atomic = &crtc->atomic; - if (!intel_crtc->active) - return; + if (atomic->disable_fbc) + intel_fbc_deactivate(crtc); - intel_crtc_wait_for_pending_flips(crtc); + if (crtc->atomic.disable_ips) + hsw_disable_ips(crtc); - intel_pre_disable_primary(crtc); + if (atomic->pre_disable_primary) + intel_pre_disable_primary(&crtc->base); + + if (atomic->disable_cxsr) { + crtc->wm.cxsr_allowed = false; + intel_set_memory_cxsr(dev_priv, false); + } +} + +static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask) +{ + struct drm_device *dev = crtc->dev; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct drm_plane *p; + int pipe = intel_crtc->pipe; intel_crtc_dpms_overlay_disable(intel_crtc); - for_each_intel_plane(dev, intel_plane) { - if (intel_plane->pipe == pipe) { - struct drm_crtc *from = intel_plane->base.crtc; - intel_plane->disable_plane(&intel_plane->base, - from ?: crtc, true); - } - } + drm_for_each_plane_mask(p, dev, plane_mask) + to_intel_plane(p)->disable_plane(p, crtc); /* * FIXME: Once we grow proper nuclear flip support out of this we need @@ -4887,12 +4864,13 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - WARN_ON(!crtc->state->enable); - - if (intel_crtc->active) + if (WARN_ON(intel_crtc->active)) return; if (intel_crtc->config->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); + + if (intel_crtc->config->has_pch_encoder) intel_prepare_shared_dpll(intel_crtc); if (intel_crtc->config->has_dp_encoder) @@ -4910,7 +4888,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) intel_crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); - intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); for_each_encoder_on_crtc(dev, crtc, encoder) if (encoder->pre_enable) @@ -4948,6 +4925,13 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) if (HAS_PCH_CPT(dev)) cpt_verify_modeset(dev, intel_crtc->pipe); + + /* Must wait for vblank to avoid spurious PCH FIFO underruns */ + if (intel_crtc->config->has_pch_encoder) + intel_wait_for_vblank(dev, pipe); + intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); + + intel_fbc_enable(intel_crtc); } /* IPS only exists on ULT machines and is tied to pipe A. */ @@ -4956,48 +4940,23 @@ static bool hsw_crtc_supports_ips(struct intel_crtc *crtc) return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A; } -/* - * This implements the workaround described in the "notes" section of the mode - * set sequence documentation. When going from no pipes or single pipe to - * multiple pipes, and planes are enabled after the pipe, we need to wait at - * least 2 vblanks on the first pipe before enabling planes on the second pipe. - */ -static void haswell_mode_set_planes_workaround(struct intel_crtc *crtc) -{ - struct drm_device *dev = crtc->base.dev; - struct intel_crtc *crtc_it, *other_active_crtc = NULL; - - /* We want to get the other_active_crtc only if there's only 1 other - * active crtc. */ - for_each_intel_crtc(dev, crtc_it) { - if (!crtc_it->active || crtc_it == crtc) - continue; - - if (other_active_crtc) - return; - - other_active_crtc = crtc_it; - } - if (!other_active_crtc) - return; - - intel_wait_for_vblank(dev, other_active_crtc->pipe); - intel_wait_for_vblank(dev, other_active_crtc->pipe); -} - static void haswell_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; - int pipe = intel_crtc->pipe; + int pipe = intel_crtc->pipe, hsw_workaround_pipe; + struct intel_crtc_state *pipe_config = + to_intel_crtc_state(crtc->state); - WARN_ON(!crtc->state->enable); - - if (intel_crtc->active) + if (WARN_ON(intel_crtc->active)) return; + if (intel_crtc->config->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, + false); + if (intel_crtc_to_shared_dpll(intel_crtc)) intel_enable_shared_dpll(intel_crtc); @@ -5022,25 +4981,26 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_crtc->active = true; - intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); - for_each_encoder_on_crtc(dev, crtc, encoder) + if (intel_crtc->config->has_pch_encoder) + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); + else + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); + + for_each_encoder_on_crtc(dev, crtc, encoder) { if (encoder->pre_enable) encoder->pre_enable(encoder); + } - if (intel_crtc->config->has_pch_encoder) { - intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, - true); + if (intel_crtc->config->has_pch_encoder) dev_priv->display.fdi_link_train(crtc); - } - intel_ddi_enable_pipe_clock(intel_crtc); + if (!intel_crtc->config->has_dsi_encoder) + intel_ddi_enable_pipe_clock(intel_crtc); - if (INTEL_INFO(dev)->gen == 9) - skylake_pfit_update(intel_crtc, 1); - else if (INTEL_INFO(dev)->gen < 9) - ironlake_pfit_enable(intel_crtc); + if (INTEL_INFO(dev)->gen >= 9) + skylake_pfit_enable(intel_crtc); else - MISSING_CASE(INTEL_INFO(dev)->gen); + ironlake_pfit_enable(intel_crtc); /* * On ILK+ LUT must be loaded before the pipe is running but with @@ -5049,7 +5009,8 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_crtc_load_lut(crtc); intel_ddi_set_pipe_settings(crtc); - intel_ddi_enable_transcoder_func(crtc); + if (!intel_crtc->config->has_dsi_encoder) + intel_ddi_enable_transcoder_func(crtc); intel_update_watermarks(crtc); intel_enable_pipe(intel_crtc); @@ -5068,12 +5029,26 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_opregion_notify_encoder(encoder, true); } + if (intel_crtc->config->has_pch_encoder) { + intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev, pipe); + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); + intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, + true); + } + /* If we change the relative order between pipe/planes enabling, we need * to change the workaround. */ - haswell_mode_set_planes_workaround(intel_crtc); + hsw_workaround_pipe = pipe_config->hsw_workaround_pipe; + if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) { + intel_wait_for_vblank(dev, hsw_workaround_pipe); + intel_wait_for_vblank(dev, hsw_workaround_pipe); + } + + intel_fbc_enable(intel_crtc); } -static void ironlake_pfit_disable(struct intel_crtc *crtc) +static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -5081,7 +5056,7 @@ static void ironlake_pfit_disable(struct intel_crtc *crtc) /* To avoid upsetting the power well on haswell only disable the pfit if * it's in use. The hw state code will make sure we get this right. */ - if (crtc->config->pch_pfit.enabled) { + if (force || crtc->config->pch_pfit.enabled) { I915_WRITE(PF_CTL(pipe), 0); I915_WRITE(PF_WIN_POS(pipe), 0); I915_WRITE(PF_WIN_SZ(pipe), 0); @@ -5095,10 +5070,9 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - u32 reg, temp; - if (!intel_crtc->active) - return; + if (intel_crtc->config->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); for_each_encoder_on_crtc(dev, crtc, encoder) encoder->disable(encoder); @@ -5106,15 +5080,22 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) drm_crtc_vblank_off(crtc); assert_vblank_disabled(crtc); + /* + * Sometimes spurious CPU pipe underruns happen when the + * pipe is already disabled, but FDI RX/TX is still enabled. + * Happens at least with VGA+HDMI cloning. Suppress them. + */ if (intel_crtc->config->has_pch_encoder) - intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); intel_disable_pipe(intel_crtc); - ironlake_pfit_disable(intel_crtc); + ironlake_pfit_disable(intel_crtc, false); - if (intel_crtc->config->has_pch_encoder) + if (intel_crtc->config->has_pch_encoder) { ironlake_fdi_disable(crtc); + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); + } for_each_encoder_on_crtc(dev, crtc, encoder) if (encoder->post_disable) @@ -5124,6 +5105,9 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) ironlake_disable_pch_transcoder(dev_priv, pipe); if (HAS_PCH_CPT(dev)) { + i915_reg_t reg; + u32 temp; + /* disable TRANS_DP_CTL */ reg = TRANS_DP_CTL(pipe); temp = I915_READ(reg); @@ -5138,18 +5122,12 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) I915_WRITE(PCH_DPLL_SEL, temp); } - /* disable PCH DPLL */ - intel_disable_shared_dpll(intel_crtc); - ironlake_fdi_pll_disable(intel_crtc); } - intel_crtc->active = false; - intel_update_watermarks(crtc); + intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); - mutex_lock(&dev->struct_mutex); - intel_fbc_update(dev); - mutex_unlock(&dev->struct_mutex); + intel_fbc_disable_crtc(intel_crtc); } static void haswell_crtc_disable(struct drm_crtc *crtc) @@ -5160,8 +5138,9 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) struct intel_encoder *encoder; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; - if (!intel_crtc->active) - return; + if (intel_crtc->config->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, + false); for_each_encoder_on_crtc(dev, crtc, encoder) { intel_opregion_notify_encoder(encoder, false); @@ -5171,24 +5150,21 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) drm_crtc_vblank_off(crtc); assert_vblank_disabled(crtc); - if (intel_crtc->config->has_pch_encoder) - intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, - false); intel_disable_pipe(intel_crtc); if (intel_crtc->config->dp_encoder_is_mst) intel_ddi_set_vc_payload_alloc(crtc, false); - intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder); + if (!intel_crtc->config->has_dsi_encoder) + intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder); - if (INTEL_INFO(dev)->gen == 9) - skylake_pfit_update(intel_crtc, 0); - else if (INTEL_INFO(dev)->gen < 9) - ironlake_pfit_disable(intel_crtc); + if (INTEL_INFO(dev)->gen >= 9) + skylake_scaler_disable(intel_crtc); else - MISSING_CASE(INTEL_INFO(dev)->gen); + ironlake_pfit_disable(intel_crtc, false); - intel_ddi_disable_pipe_clock(intel_crtc); + if (!intel_crtc->config->has_dsi_encoder) + intel_ddi_disable_pipe_clock(intel_crtc); if (intel_crtc->config->has_pch_encoder) { lpt_disable_pch_transcoder(dev_priv); @@ -5199,24 +5175,13 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) if (encoder->post_disable) encoder->post_disable(encoder); - intel_crtc->active = false; - intel_update_watermarks(crtc); - - mutex_lock(&dev->struct_mutex); - intel_fbc_update(dev); - mutex_unlock(&dev->struct_mutex); - - if (intel_crtc_to_shared_dpll(intel_crtc)) - intel_disable_shared_dpll(intel_crtc); -} + if (intel_crtc->config->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, + true); -static void ironlake_crtc_off(struct drm_crtc *crtc) -{ - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - intel_put_shared_dpll(intel_crtc); + intel_fbc_disable_crtc(intel_crtc); } - static void i9xx_pfit_enable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -5245,22 +5210,40 @@ static enum intel_display_power_domain port_to_power_domain(enum port port) { switch (port) { case PORT_A: - return POWER_DOMAIN_PORT_DDI_A_4_LANES; + return POWER_DOMAIN_PORT_DDI_A_LANES; case PORT_B: - return POWER_DOMAIN_PORT_DDI_B_4_LANES; + return POWER_DOMAIN_PORT_DDI_B_LANES; case PORT_C: - return POWER_DOMAIN_PORT_DDI_C_4_LANES; + return POWER_DOMAIN_PORT_DDI_C_LANES; case PORT_D: - return POWER_DOMAIN_PORT_DDI_D_4_LANES; + return POWER_DOMAIN_PORT_DDI_D_LANES; + case PORT_E: + return POWER_DOMAIN_PORT_DDI_E_LANES; default: - WARN_ON_ONCE(1); + MISSING_CASE(port); return POWER_DOMAIN_PORT_OTHER; } } -#define for_each_power_domain(domain, mask) \ - for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ - if ((1 << (domain)) & (mask)) +static enum intel_display_power_domain port_to_aux_power_domain(enum port port) +{ + switch (port) { + case PORT_A: + return POWER_DOMAIN_AUX_A; + case PORT_B: + return POWER_DOMAIN_AUX_B; + case PORT_C: + return POWER_DOMAIN_AUX_C; + case PORT_D: + return POWER_DOMAIN_AUX_D; + case PORT_E: + /* FIXME: Check VBT for actual wiring of PORT E */ + return POWER_DOMAIN_AUX_D; + default: + MISSING_CASE(port); + return POWER_DOMAIN_AUX_A; + } +} enum intel_display_power_domain intel_display_port_power_domain(struct intel_encoder *intel_encoder) @@ -5289,6 +5272,36 @@ intel_display_port_power_domain(struct intel_encoder *intel_encoder) } } +enum intel_display_power_domain +intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder) +{ + struct drm_device *dev = intel_encoder->base.dev; + struct intel_digital_port *intel_dig_port; + + switch (intel_encoder->type) { + case INTEL_OUTPUT_UNKNOWN: + case INTEL_OUTPUT_HDMI: + /* + * Only DDI platforms should ever use these output types. + * We can get here after the HDMI detect code has already set + * the type of the shared encoder. Since we can't be sure + * what's the status of the given connectors, play safe and + * run the DP detection too. + */ + WARN_ON_ONCE(!HAS_DDI(dev)); + case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_EDP: + intel_dig_port = enc_to_dig_port(&intel_encoder->base); + return port_to_aux_power_domain(intel_dig_port->port); + case INTEL_OUTPUT_DP_MST: + intel_dig_port = enc_to_mst(&intel_encoder->base)->primary; + return port_to_aux_power_domain(intel_dig_port->port); + default: + MISSING_CASE(intel_encoder->type); + return POWER_DOMAIN_AUX_A; + } +} + static unsigned long get_crtc_power_domains(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; @@ -5296,9 +5309,10 @@ static unsigned long get_crtc_power_domains(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; unsigned long mask; - enum transcoder transcoder; + enum transcoder transcoder = intel_crtc->config->cpu_transcoder; - transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe); + if (!crtc->state->active) + return 0; mask = BIT(POWER_DOMAIN_PIPE(pipe)); mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder)); @@ -5312,45 +5326,151 @@ static unsigned long get_crtc_power_domains(struct drm_crtc *crtc) return mask; } +static unsigned long modeset_get_crtc_power_domains(struct drm_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + enum intel_display_power_domain domain; + unsigned long domains, new_domains, old_domains; + + old_domains = intel_crtc->enabled_power_domains; + intel_crtc->enabled_power_domains = new_domains = get_crtc_power_domains(crtc); + + domains = new_domains & ~old_domains; + + for_each_power_domain(domain, domains) + intel_display_power_get(dev_priv, domain); + + return old_domains & ~new_domains; +} + +static void modeset_put_power_domains(struct drm_i915_private *dev_priv, + unsigned long domains) +{ + enum intel_display_power_domain domain; + + for_each_power_domain(domain, domains) + intel_display_power_put(dev_priv, domain); +} + static void modeset_update_crtc_power_domains(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long pipe_domains[I915_MAX_PIPES] = { 0, }; - struct intel_crtc *crtc; + unsigned long put_domains[I915_MAX_PIPES] = {}; + struct drm_crtc_state *crtc_state; + struct drm_crtc *crtc; + int i; - /* - * First get all needed power domains, then put all unneeded, to avoid - * any unnecessary toggling of the power wells. - */ - for_each_intel_crtc(dev, crtc) { - enum intel_display_power_domain domain; + for_each_crtc_in_state(state, crtc, crtc_state, i) { + if (needs_modeset(crtc->state)) + put_domains[to_intel_crtc(crtc)->pipe] = + modeset_get_crtc_power_domains(crtc); + } - if (!crtc->base.state->enable) - continue; + if (dev_priv->display.modeset_commit_cdclk) { + unsigned int cdclk = to_intel_atomic_state(state)->cdclk; + + if (cdclk != dev_priv->cdclk_freq && + !WARN_ON(!state->allow_modeset)) + dev_priv->display.modeset_commit_cdclk(state); + } + + for (i = 0; i < I915_MAX_PIPES; i++) + if (put_domains[i]) + modeset_put_power_domains(dev_priv, put_domains[i]); +} + +static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) +{ + int max_cdclk_freq = dev_priv->max_cdclk_freq; + + if (INTEL_INFO(dev_priv)->gen >= 9 || + IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) + return max_cdclk_freq; + else if (IS_CHERRYVIEW(dev_priv)) + return max_cdclk_freq*95/100; + else if (INTEL_INFO(dev_priv)->gen < 4) + return 2*max_cdclk_freq*90/100; + else + return max_cdclk_freq*90/100; +} - pipe_domains[crtc->pipe] = get_crtc_power_domains(&crtc->base); +static void intel_update_max_cdclk(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK; - for_each_power_domain(domain, pipe_domains[crtc->pipe]) - intel_display_power_get(dev_priv, domain); + if (limit == SKL_DFSM_CDCLK_LIMIT_675) + dev_priv->max_cdclk_freq = 675000; + else if (limit == SKL_DFSM_CDCLK_LIMIT_540) + dev_priv->max_cdclk_freq = 540000; + else if (limit == SKL_DFSM_CDCLK_LIMIT_450) + dev_priv->max_cdclk_freq = 450000; + else + dev_priv->max_cdclk_freq = 337500; + } else if (IS_BROADWELL(dev)) { + /* + * FIXME with extra cooling we can allow + * 540 MHz for ULX and 675 Mhz for ULT. + * How can we know if extra cooling is + * available? PCI ID, VTB, something else? + */ + if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) + dev_priv->max_cdclk_freq = 450000; + else if (IS_BDW_ULX(dev)) + dev_priv->max_cdclk_freq = 450000; + else if (IS_BDW_ULT(dev)) + dev_priv->max_cdclk_freq = 540000; + else + dev_priv->max_cdclk_freq = 675000; + } else if (IS_CHERRYVIEW(dev)) { + dev_priv->max_cdclk_freq = 320000; + } else if (IS_VALLEYVIEW(dev)) { + dev_priv->max_cdclk_freq = 400000; + } else { + /* otherwise assume cdclk is fixed */ + dev_priv->max_cdclk_freq = dev_priv->cdclk_freq; } - if (dev_priv->display.modeset_global_resources) - dev_priv->display.modeset_global_resources(state); + dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv); - for_each_intel_crtc(dev, crtc) { - enum intel_display_power_domain domain; + DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n", + dev_priv->max_cdclk_freq); - for_each_power_domain(domain, crtc->enabled_power_domains) - intel_display_power_put(dev_priv, domain); + DRM_DEBUG_DRIVER("Max dotclock rate: %d kHz\n", + dev_priv->max_dotclk_freq); +} + +static void intel_update_cdclk(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev); + DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n", + dev_priv->cdclk_freq); - crtc->enabled_power_domains = pipe_domains[crtc->pipe]; + /* + * Program the gmbus_freq based on the cdclk frequency. + * BSpec erroneously claims we should aim for 4MHz, but + * in fact 1MHz is the correct frequency. + */ + if (IS_VALLEYVIEW(dev)) { + /* + * Program the gmbus_freq based on the cdclk frequency. + * BSpec erroneously claims we should aim for 4MHz, but + * in fact 1MHz is the correct frequency. + */ + I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000)); } - intel_display_set_init_power(dev_priv, false); + if (dev_priv->max_cdclk_freq == 0) + intel_update_max_cdclk(dev); } -void broxton_set_cdclk(struct drm_device *dev, int frequency) +static void broxton_set_cdclk(struct drm_device *dev, int frequency) { struct drm_i915_private *dev_priv = dev->dev_private; uint32_t divider; @@ -5466,7 +5586,7 @@ void broxton_set_cdclk(struct drm_device *dev, int frequency) return; } - dev_priv->cdclk_freq = frequency; + intel_update_cdclk(dev); } void broxton_init_cdclk(struct drm_device *dev) @@ -5641,6 +5761,7 @@ static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv) static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq) { + struct drm_device *dev = dev_priv->dev; u32 freq_select, pcu_ack; DRM_DEBUG_DRIVER("Changing CDCLK to %dKHz\n", freq); @@ -5681,6 +5802,8 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq) mutex_lock(&dev_priv->rps.hw_lock); sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack); mutex_unlock(&dev_priv->rps.hw_lock); + + intel_update_cdclk(dev); } void skl_uninit_cdclk(struct drm_i915_private *dev_priv) @@ -5698,32 +5821,19 @@ void skl_uninit_cdclk(struct drm_i915_private *dev_priv) I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE); if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1)) DRM_ERROR("Couldn't disable DPLL0\n"); - - intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS); } void skl_init_cdclk(struct drm_i915_private *dev_priv) { - u32 val; unsigned int required_vco; - /* enable PCH reset handshake */ - val = I915_READ(HSW_NDE_RSTWRN_OPT); - I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE); - - /* enable PG1 and Misc I/O */ - intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS); - - /* DPLL0 already enabed !? */ - if (I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE) { - DRM_DEBUG_DRIVER("DPLL0 already running\n"); - return; + /* DPLL0 not enabled (happens on early BIOS versions) */ + if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) { + /* enable DPLL0 */ + required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk); + skl_dpll0_enable(dev_priv, required_vco); } - /* enable DPLL0 */ - required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk); - skl_dpll0_enable(dev_priv, required_vco); - /* set CDCLK to the frequency the BIOS chose */ skl_set_cdclk(dev_priv, dev_priv->skl_boot_cdclk); @@ -5737,34 +5847,43 @@ void skl_init_cdclk(struct drm_i915_private *dev_priv) DRM_ERROR("DBuf power enable timeout\n"); } -/* returns HPLL frequency in kHz */ -static int valleyview_get_vco(struct drm_i915_private *dev_priv) +int skl_sanitize_cdclk(struct drm_i915_private *dev_priv) { - int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 }; - - /* Obtain SKU information */ - mutex_lock(&dev_priv->sb_lock); - hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) & - CCK_FUSE_HPLL_FREQ_MASK; - mutex_unlock(&dev_priv->sb_lock); - - return vco_freq[hpll_freq] * 1000; -} + uint32_t lcpll1 = I915_READ(LCPLL1_CTL); + uint32_t cdctl = I915_READ(CDCLK_CTL); + int freq = dev_priv->skl_boot_cdclk; -static void vlv_update_cdclk(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; + /* + * check if the pre-os intialized the display + * There is SWF18 scratchpad register defined which is set by the + * pre-os which can be used by the OS drivers to check the status + */ + if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0) + goto sanitize; - dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev); - DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n", - dev_priv->cdclk_freq); + /* Is PLL enabled and locked ? */ + if (!((lcpll1 & LCPLL_PLL_ENABLE) && (lcpll1 & LCPLL_PLL_LOCK))) + goto sanitize; - /* - * Program the gmbus_freq based on the cdclk frequency. - * BSpec erroneously claims we should aim for 4MHz, but - * in fact 1MHz is the correct frequency. + /* DPLL okay; verify the cdclock + * + * Noticed in some instances that the freq selection is correct but + * decimal part is programmed wrong from BIOS where pre-os does not + * enable display. Verify the same as well. */ - I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000)); + if (cdctl == ((cdctl & CDCLK_FREQ_SEL_MASK) | skl_cdclk_decimal(freq))) + /* All well; nothing to sanitize */ + return false; +sanitize: + /* + * As of now initialize with max cdclk till + * we get dynamic cdclk support + * */ + dev_priv->skl_boot_cdclk = dev_priv->max_cdclk_freq; + skl_init_cdclk(dev_priv); + + /* we did have to sanitize */ + return true; } /* Adjust CDclk dividers to allow high res or save power if possible */ @@ -5804,12 +5923,12 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) /* adjust cdclk divider */ val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL); - val &= ~DISPLAY_FREQUENCY_VALUES; + val &= ~CCK_FREQUENCY_VALUES; val |= divider; vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val); if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) & - DISPLAY_FREQUENCY_STATUS) == (divider << DISPLAY_FREQUENCY_STATUS_SHIFT), + CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT), 50)) DRM_ERROR("timed out waiting for CDclk change\n"); } @@ -5830,7 +5949,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) mutex_unlock(&dev_priv->sb_lock); - vlv_update_cdclk(dev); + intel_update_cdclk(dev); } static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) @@ -5871,7 +5990,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) } mutex_unlock(&dev_priv->rps.hw_lock); - vlv_update_cdclk(dev); + intel_update_cdclk(dev); } static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, @@ -5934,11 +6053,7 @@ static int intel_mode_max_pixclk(struct drm_device *dev, int max_pixclk = 0; for_each_intel_crtc(dev, intel_crtc) { - if (state) - crtc_state = - intel_atomic_get_crtc_state(state, intel_crtc); - else - crtc_state = intel_crtc->config; + crtc_state = intel_atomic_get_crtc_state(state, intel_crtc); if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); @@ -5952,39 +6067,32 @@ static int intel_mode_max_pixclk(struct drm_device *dev, return max_pixclk; } -static int valleyview_modeset_global_pipes(struct drm_atomic_state *state) +static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state) { - struct drm_i915_private *dev_priv = to_i915(state->dev); - struct drm_crtc *crtc; - struct drm_crtc_state *crtc_state; - int max_pixclk = intel_mode_max_pixclk(state->dev, state); - int cdclk, i; + struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int max_pixclk = intel_mode_max_pixclk(dev, state); if (max_pixclk < 0) return max_pixclk; - if (IS_VALLEYVIEW(dev_priv)) - cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk); - else - cdclk = broxton_calc_cdclk(dev_priv, max_pixclk); + to_intel_atomic_state(state)->cdclk = + valleyview_calc_cdclk(dev_priv, max_pixclk); - if (cdclk == dev_priv->cdclk_freq) - return 0; + return 0; +} - /* add all active pipes to the state */ - for_each_crtc(state->dev, crtc) { - if (!crtc->state->enable) - continue; +static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state) +{ + struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int max_pixclk = intel_mode_max_pixclk(dev, state); - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); - } + if (max_pixclk < 0) + return max_pixclk; - /* disable/enable all currently active pipes while we change cdclk */ - for_each_crtc_in_state(state, crtc, crtc_state, i) - if (crtc_state->enable) - crtc_state->mode_changed = true; + to_intel_atomic_state(state)->cdclk = + broxton_calc_cdclk(dev_priv, max_pixclk); return 0; } @@ -5998,10 +6106,10 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv) else default_credits = PFI_CREDIT(8); - if (DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 1000) >= dev_priv->rps.cz_freq) { + if (dev_priv->cdclk_freq >= dev_priv->czclk_freq) { /* CHV suggested value is 31 or 63 */ if (IS_CHERRYVIEW(dev_priv)) - credits = PFI_CREDIT_31; + credits = PFI_CREDIT_63; else credits = PFI_CREDIT(15); } else { @@ -6025,41 +6133,31 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv) WARN_ON(I915_READ(GCI_CONTROL) & PFI_CREDIT_RESEND); } -static void valleyview_modeset_global_resources(struct drm_atomic_state *old_state) +static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state) { struct drm_device *dev = old_state->dev; + unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk; struct drm_i915_private *dev_priv = dev->dev_private; - int max_pixclk = intel_mode_max_pixclk(dev, NULL); - int req_cdclk; - - /* The path in intel_mode_max_pixclk() with a NULL atomic state should - * never fail. */ - if (WARN_ON(max_pixclk < 0)) - return; - req_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk); - - if (req_cdclk != dev_priv->cdclk_freq) { - /* - * FIXME: We can end up here with all power domains off, yet - * with a CDCLK frequency other than the minimum. To account - * for this take the PIPE-A power domain, which covers the HW - * blocks needed for the following programming. This can be - * removed once it's guaranteed that we get here either with - * the minimum CDCLK set, or the required power domains - * enabled. - */ - intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); + /* + * FIXME: We can end up here with all power domains off, yet + * with a CDCLK frequency other than the minimum. To account + * for this take the PIPE-A power domain, which covers the HW + * blocks needed for the following programming. This can be + * removed once it's guaranteed that we get here either with + * the minimum CDCLK set, or the required power domains + * enabled. + */ + intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A); - if (IS_CHERRYVIEW(dev)) - cherryview_set_cdclk(dev, req_cdclk); - else - valleyview_set_cdclk(dev, req_cdclk); + if (IS_CHERRYVIEW(dev)) + cherryview_set_cdclk(dev, req_cdclk); + else + valleyview_set_cdclk(dev, req_cdclk); - vlv_program_pfi_credits(dev_priv); + vlv_program_pfi_credits(dev_priv); - intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A); - } + intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A); } static void valleyview_crtc_enable(struct drm_crtc *crtc) @@ -6069,22 +6167,10 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - bool is_dsi; - - WARN_ON(!crtc->state->enable); - if (intel_crtc->active) + if (WARN_ON(intel_crtc->active)) return; - is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI); - - if (!is_dsi) { - if (IS_CHERRYVIEW(dev)) - chv_prepare_pll(intel_crtc, intel_crtc->config); - else - vlv_prepare_pll(intel_crtc, intel_crtc->config); - } - if (intel_crtc->config->has_dp_encoder) intel_dp_set_m_n(intel_crtc, M1_N1); @@ -6107,11 +6193,14 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) if (encoder->pre_pll_enable) encoder->pre_pll_enable(encoder); - if (!is_dsi) { - if (IS_CHERRYVIEW(dev)) + if (!intel_crtc->config->has_dsi_encoder) { + if (IS_CHERRYVIEW(dev)) { + chv_prepare_pll(intel_crtc, intel_crtc->config); chv_enable_pll(intel_crtc, intel_crtc->config); - else + } else { + vlv_prepare_pll(intel_crtc, intel_crtc->config); vlv_enable_pll(intel_crtc, intel_crtc->config); + } } for_each_encoder_on_crtc(dev, crtc, encoder) @@ -6122,7 +6211,6 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) intel_crtc_load_lut(crtc); - intel_update_watermarks(crtc); intel_enable_pipe(intel_crtc); assert_vblank_disabled(crtc); @@ -6149,9 +6237,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - WARN_ON(!crtc->state->enable); - - if (intel_crtc->active) + if (WARN_ON(intel_crtc->active)) return; i9xx_set_pll_dividers(intel_crtc); @@ -6186,6 +6272,8 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) for_each_encoder_on_crtc(dev, crtc, encoder) encoder->enable(encoder); + + intel_fbc_enable(intel_crtc); } static void i9xx_pfit_disable(struct intel_crtc *crtc) @@ -6211,9 +6299,6 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - if (!intel_crtc->active) - return; - /* * On gen2 planes are double buffered but the pipe isn't, so we must * wait for planes to fully turn off before disabling the pipe. @@ -6236,7 +6321,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) if (encoder->post_disable) encoder->post_disable(encoder); - if (!intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) { + if (!intel_crtc->config->has_dsi_encoder) { if (IS_CHERRYVIEW(dev)) chv_disable_pll(dev_priv, pipe); else if (IS_VALLEYVIEW(dev)) @@ -6245,93 +6330,100 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) i9xx_disable_pll(intel_crtc); } + for_each_encoder_on_crtc(dev, crtc, encoder) + if (encoder->post_pll_disable) + encoder->post_pll_disable(encoder); + if (!IS_GEN2(dev)) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); - intel_crtc->active = false; - intel_update_watermarks(crtc); - - mutex_lock(&dev->struct_mutex); - intel_fbc_update(dev); - mutex_unlock(&dev->struct_mutex); -} - -static void i9xx_crtc_off(struct drm_crtc *crtc) -{ + intel_fbc_disable_crtc(intel_crtc); } -/* Master function to enable/disable CRTC and corresponding power wells */ -void intel_crtc_control(struct drm_crtc *crtc, bool enable) +static void intel_crtc_disable_noatomic(struct drm_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); enum intel_display_power_domain domain; unsigned long domains; - if (enable) { - if (!intel_crtc->active) { - domains = get_crtc_power_domains(crtc); - for_each_power_domain(domain, domains) - intel_display_power_get(dev_priv, domain); - intel_crtc->enabled_power_domains = domains; + if (!intel_crtc->active) + return; - dev_priv->display.crtc_enable(crtc); - intel_crtc_enable_planes(crtc); - } - } else { - if (intel_crtc->active) { - intel_crtc_disable_planes(crtc); - dev_priv->display.crtc_disable(crtc); + if (to_intel_plane_state(crtc->primary->state)->visible) { + WARN_ON(intel_crtc->unpin_work); - domains = intel_crtc->enabled_power_domains; - for_each_power_domain(domain, domains) - intel_display_power_put(dev_priv, domain); - intel_crtc->enabled_power_domains = 0; - } + intel_pre_disable_primary(crtc); } + + intel_crtc_disable_planes(crtc, crtc->state->plane_mask); + dev_priv->display.crtc_disable(crtc); + intel_crtc->active = false; + intel_update_watermarks(crtc); + intel_disable_shared_dpll(intel_crtc); + + domains = intel_crtc->enabled_power_domains; + for_each_power_domain(domain, domains) + intel_display_power_put(dev_priv, domain); + intel_crtc->enabled_power_domains = 0; } -/** - * Sets the power management mode of the pipe and plane. +/* + * turn all crtc's off, but do not adjust state + * This has to be paired with a call to intel_modeset_setup_hw_state. */ -void intel_crtc_update_dpms(struct drm_crtc *crtc) +int intel_display_suspend(struct drm_device *dev) { - struct drm_device *dev = crtc->dev; - struct intel_encoder *intel_encoder; - bool enable = false; - - for_each_encoder_on_crtc(dev, crtc, intel_encoder) - enable |= intel_encoder->connectors_active; + struct drm_mode_config *config = &dev->mode_config; + struct drm_modeset_acquire_ctx *ctx = config->acquire_ctx; + struct drm_atomic_state *state; + struct drm_crtc *crtc; + unsigned crtc_mask = 0; + int ret = 0; - intel_crtc_control(crtc, enable); + if (WARN_ON(!ctx)) + return 0; - crtc->state->active = enable; -} + lockdep_assert_held(&ctx->ww_ctx); + state = drm_atomic_state_alloc(dev); + if (WARN_ON(!state)) + return -ENOMEM; -static void intel_crtc_disable(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_connector *connector; - struct drm_i915_private *dev_priv = dev->dev_private; + state->acquire_ctx = ctx; + state->allow_modeset = true; - intel_crtc_disable_planes(crtc); - dev_priv->display.crtc_disable(crtc); - dev_priv->display.off(crtc); + for_each_crtc(dev, crtc) { + struct drm_crtc_state *crtc_state = + drm_atomic_get_crtc_state(state, crtc); - drm_plane_helper_disable(crtc->primary); + ret = PTR_ERR_OR_ZERO(crtc_state); + if (ret) + goto free; - /* Update computed state. */ - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - if (!connector->encoder || !connector->encoder->crtc) + if (!crtc_state->active) continue; - if (connector->encoder->crtc != crtc) - continue; + crtc_state->active = false; + crtc_mask |= 1 << drm_crtc_index(crtc); + } - connector->dpms = DRM_MODE_DPMS_OFF; - to_intel_encoder(connector->encoder)->connectors_active = false; + if (crtc_mask) { + ret = drm_atomic_commit(state); + + if (!ret) { + for_each_crtc(dev, crtc) + if (crtc_mask & (1 << drm_crtc_index(crtc))) + crtc->state->active = true; + + return ret; + } } + +free: + if (ret) + DRM_ERROR("Suspending crtc's failed with %i\n", ret); + drm_atomic_state_free(state); + return ret; } void intel_encoder_destroy(struct drm_encoder *encoder) @@ -6342,62 +6434,42 @@ void intel_encoder_destroy(struct drm_encoder *encoder) kfree(intel_encoder); } -/* Simple dpms helper for encoders with just one connector, no cloning and only - * one kind of off state. It clamps all !ON modes to fully OFF and changes the - * state of the entire output pipe. */ -static void intel_encoder_dpms(struct intel_encoder *encoder, int mode) -{ - if (mode == DRM_MODE_DPMS_ON) { - encoder->connectors_active = true; - - intel_crtc_update_dpms(encoder->base.crtc); - } else { - encoder->connectors_active = false; - - intel_crtc_update_dpms(encoder->base.crtc); - } -} - /* Cross check the actual hw state with our own modeset state tracking (and it's * internal consistency). */ static void intel_connector_check_state(struct intel_connector *connector) { + struct drm_crtc *crtc = connector->base.state->crtc; + + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", + connector->base.base.id, + connector->base.name); + if (connector->get_hw_state(connector)) { struct intel_encoder *encoder = connector->encoder; - struct drm_crtc *crtc; - bool encoder_enabled; - enum pipe pipe; + struct drm_connector_state *conn_state = connector->base.state; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.base.id, - connector->base.name); + I915_STATE_WARN(!crtc, + "connector enabled without attached crtc\n"); - /* there is no real hw state for MST connectors */ - if (connector->mst_port) + if (!crtc) return; - I915_STATE_WARN(connector->base.dpms == DRM_MODE_DPMS_OFF, - "wrong connector dpms state\n"); - I915_STATE_WARN(connector->base.encoder != &encoder->base, - "active connector not linked to encoder\n"); - - if (encoder) { - I915_STATE_WARN(!encoder->connectors_active, - "encoder->connectors_active not set\n"); + I915_STATE_WARN(!crtc->state->active, + "connector is active, but attached crtc isn't\n"); - encoder_enabled = encoder->get_hw_state(encoder, &pipe); - I915_STATE_WARN(!encoder_enabled, "encoder not enabled\n"); - if (I915_STATE_WARN_ON(!encoder->base.crtc)) - return; + if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST) + return; - crtc = encoder->base.crtc; + I915_STATE_WARN(conn_state->best_encoder != &encoder->base, + "atomic encoder doesn't match attached encoder\n"); - I915_STATE_WARN(!crtc->state->enable, - "crtc not enabled\n"); - I915_STATE_WARN(!to_intel_crtc(crtc)->active, "crtc not active\n"); - I915_STATE_WARN(pipe != to_intel_crtc(crtc)->pipe, - "encoder active on the wrong pipe\n"); - } + I915_STATE_WARN(conn_state->crtc != encoder->base.crtc, + "attached encoder crtc differs from connector crtc\n"); + } else { + I915_STATE_WARN(crtc && crtc->state->active, + "attached crtc is active, but connector isn't\n"); + I915_STATE_WARN(!crtc && connector->base.state->best_encoder, + "best encoder set without crtc!\n"); } } @@ -6429,26 +6501,6 @@ struct intel_connector *intel_connector_alloc(void) return connector; } -/* Even simpler default implementation, if there's really no special case to - * consider. */ -void intel_connector_dpms(struct drm_connector *connector, int mode) -{ - /* All the simple cases only support two dpms states. */ - if (mode != DRM_MODE_DPMS_ON) - mode = DRM_MODE_DPMS_OFF; - - if (mode == connector->dpms) - return; - - connector->dpms = mode; - - /* Only need to change hw state when actually enabled */ - if (connector->encoder) - intel_encoder_dpms(to_intel_encoder(connector->encoder), mode); - - intel_modeset_check_state(connector->dev); -} - /* Simple connector->get_hw_state implementation for encoders that support only * one connector and no cloning and hence the encoder state determines the state * of the connector. */ @@ -6544,7 +6596,7 @@ static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; int lane, link_bw, fdi_dotclock, ret; bool needs_recompute = false; @@ -6586,12 +6638,45 @@ retry: return ret; } +static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv, + struct intel_crtc_state *pipe_config) +{ + if (pipe_config->pipe_bpp > 24) + return false; + + /* HSW can handle pixel rate up to cdclk? */ + if (IS_HASWELL(dev_priv->dev)) + return true; + + /* + * We compare against max which means we must take + * the increased cdclk requirement into account when + * calculating the new cdclk. + * + * Should measure whether using a lower cdclk w/o IPS + */ + return ilk_pipe_pixel_rate(pipe_config) <= + dev_priv->max_cdclk_freq * 95 / 100; +} + static void hsw_compute_ips_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + pipe_config->ips_enabled = i915.enable_ips && - hsw_crtc_supports_ips(crtc) && - pipe_config->pipe_bpp <= 24; + hsw_crtc_supports_ips(crtc) && + pipe_config_supports_ips(dev_priv, pipe_config); +} + +static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc) +{ + const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + + /* GDG double wide on either pipe, otherwise pipe A only */ + return INTEL_INFO(dev_priv)->gen < 4 && + (crtc->pipe == PIPE_A || IS_I915G(dev_priv)); } static int intel_crtc_compute_config(struct intel_crtc *crtc, @@ -6599,29 +6684,28 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; - int ret; + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; /* FIXME should check pixel clock limits on all platforms */ if (INTEL_INFO(dev)->gen < 4) { - int clock_limit = - dev_priv->display.get_display_clock_speed(dev); + int clock_limit = dev_priv->max_cdclk_freq * 9 / 10; /* - * Enable pixel doubling when the dot clock + * Enable double wide mode when the dot clock * is > 90% of the (display) core speed. - * - * GDG double wide on either pipe, - * otherwise pipe A only. */ - if ((crtc->pipe == PIPE_A || IS_I915G(dev)) && - adjusted_mode->crtc_clock > clock_limit * 9 / 10) { + if (intel_crtc_supports_double_wide(crtc) && + adjusted_mode->crtc_clock > clock_limit) { clock_limit *= 2; pipe_config->double_wide = true; } - if (adjusted_mode->crtc_clock > clock_limit * 9 / 10) + if (adjusted_mode->crtc_clock > clock_limit) { + DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n", + adjusted_mode->crtc_clock, clock_limit, + yesno(pipe_config->double_wide)); return -EINVAL; + } } /* @@ -6638,7 +6722,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw. */ if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) && - adjusted_mode->hsync_start == adjusted_mode->hdisplay) + adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay) return -EINVAL; if (HAS_IPS(dev)) @@ -6647,14 +6731,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, if (pipe_config->has_pch_encoder) return ironlake_fdi_compute_config(crtc, pipe_config); - /* FIXME: remove below call once atomic mode set is place and all crtc - * related checks called from atomic_crtc_check function */ - ret = 0; - DRM_DEBUG_KMS("intel_crtc = %p drm_state (pipe_config->base.state) = %p\n", - crtc, pipe_config->base.state); - ret = intel_atomic_setup_scalers(dev, crtc, pipe_config); - - return ret; + return 0; } static int skylake_get_display_clock_speed(struct drm_device *dev) @@ -6664,10 +6741,8 @@ static int skylake_get_display_clock_speed(struct drm_device *dev) uint32_t cdctl = I915_READ(CDCLK_CTL); uint32_t linkrate; - if (!(lcpll1 & LCPLL_PLL_ENABLE)) { - WARN(1, "LCPLL1 not enabled\n"); + if (!(lcpll1 & LCPLL_PLL_ENABLE)) return 24000; /* 24MHz is the cd freq with NSSC ref */ - } if ((cdctl & CDCLK_FREQ_SEL_MASK) == CDCLK_FREQ_540) return 540000; @@ -6706,6 +6781,34 @@ static int skylake_get_display_clock_speed(struct drm_device *dev) return 24000; } +static int broxton_get_display_clock_speed(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + uint32_t cdctl = I915_READ(CDCLK_CTL); + uint32_t pll_ratio = I915_READ(BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK; + uint32_t pll_enab = I915_READ(BXT_DE_PLL_ENABLE); + int cdclk; + + if (!(pll_enab & BXT_DE_PLL_PLL_ENABLE)) + return 19200; + + cdclk = 19200 * pll_ratio / 2; + + switch (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) { + case BXT_CDCLK_CD2X_DIV_SEL_1: + return cdclk; /* 576MHz or 624MHz */ + case BXT_CDCLK_CD2X_DIV_SEL_1_5: + return cdclk * 2 / 3; /* 384MHz */ + case BXT_CDCLK_CD2X_DIV_SEL_2: + return cdclk / 2; /* 288MHz */ + case BXT_CDCLK_CD2X_DIV_SEL_4: + return cdclk / 4; /* 144MHz */ + } + + /* error case, do as if DE PLL isn't enabled */ + return 19200; +} + static int broadwell_get_display_clock_speed(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -6746,24 +6849,8 @@ static int haswell_get_display_clock_speed(struct drm_device *dev) static int valleyview_get_display_clock_speed(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - u32 val; - int divider; - - if (dev_priv->hpll_freq == 0) - dev_priv->hpll_freq = valleyview_get_vco(dev_priv); - - mutex_lock(&dev_priv->sb_lock); - val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL); - mutex_unlock(&dev_priv->sb_lock); - - divider = val & DISPLAY_FREQUENCY_VALUES; - - WARN((val & DISPLAY_FREQUENCY_STATUS) != - (divider << DISPLAY_FREQUENCY_STATUS_SHIFT), - "cdclk change in progress\n"); - - return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1); + return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk", + CCK_DISPLAY_CLOCK_CONTROL); } static int ilk_get_display_clock_speed(struct drm_device *dev) @@ -6834,20 +6921,37 @@ static int i865_get_display_clock_speed(struct drm_device *dev) return 266667; } -static int i855_get_display_clock_speed(struct drm_device *dev) +static int i85x_get_display_clock_speed(struct drm_device *dev) { u16 hpllcc = 0; + + /* + * 852GM/852GMV only supports 133 MHz and the HPLLCC + * encoding is different :( + * FIXME is this the right way to detect 852GM/852GMV? + */ + if (dev->pdev->revision == 0x1) + return 133333; + + pci_bus_read_config_word(dev->pdev->bus, + PCI_DEVFN(0, 3), HPLLCC, &hpllcc); + /* Assume that the hardware is in the high speed state. This * should be the default. */ switch (hpllcc & GC_CLOCK_CONTROL_MASK) { case GC_CLOCK_133_200: + case GC_CLOCK_133_200_2: case GC_CLOCK_100_200: return 200000; case GC_CLOCK_166_250: return 250000; case GC_CLOCK_100_133: return 133333; + case GC_CLOCK_133_266: + case GC_CLOCK_133_266_2: + case GC_CLOCK_166_266: + return 266667; } /* Shouldn't happen */ @@ -6859,6 +6963,175 @@ static int i830_get_display_clock_speed(struct drm_device *dev) return 133333; } +static unsigned int intel_hpll_vco(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + static const unsigned int blb_vco[8] = { + [0] = 3200000, + [1] = 4000000, + [2] = 5333333, + [3] = 4800000, + [4] = 6400000, + }; + static const unsigned int pnv_vco[8] = { + [0] = 3200000, + [1] = 4000000, + [2] = 5333333, + [3] = 4800000, + [4] = 2666667, + }; + static const unsigned int cl_vco[8] = { + [0] = 3200000, + [1] = 4000000, + [2] = 5333333, + [3] = 6400000, + [4] = 3333333, + [5] = 3566667, + [6] = 4266667, + }; + static const unsigned int elk_vco[8] = { + [0] = 3200000, + [1] = 4000000, + [2] = 5333333, + [3] = 4800000, + }; + static const unsigned int ctg_vco[8] = { + [0] = 3200000, + [1] = 4000000, + [2] = 5333333, + [3] = 6400000, + [4] = 2666667, + [5] = 4266667, + }; + const unsigned int *vco_table; + unsigned int vco; + uint8_t tmp = 0; + + /* FIXME other chipsets? */ + if (IS_GM45(dev)) + vco_table = ctg_vco; + else if (IS_G4X(dev)) + vco_table = elk_vco; + else if (IS_CRESTLINE(dev)) + vco_table = cl_vco; + else if (IS_PINEVIEW(dev)) + vco_table = pnv_vco; + else if (IS_G33(dev)) + vco_table = blb_vco; + else + return 0; + + tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO); + + vco = vco_table[tmp & 0x7]; + if (vco == 0) + DRM_ERROR("Bad HPLL VCO (HPLLVCO=0x%02x)\n", tmp); + else + DRM_DEBUG_KMS("HPLL VCO %u kHz\n", vco); + + return vco; +} + +static int gm45_get_display_clock_speed(struct drm_device *dev) +{ + unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + uint16_t tmp = 0; + + pci_read_config_word(dev->pdev, GCFGC, &tmp); + + cdclk_sel = (tmp >> 12) & 0x1; + + switch (vco) { + case 2666667: + case 4000000: + case 5333333: + return cdclk_sel ? 333333 : 222222; + case 3200000: + return cdclk_sel ? 320000 : 228571; + default: + DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n", vco, tmp); + return 222222; + } +} + +static int i965gm_get_display_clock_speed(struct drm_device *dev) +{ + static const uint8_t div_3200[] = { 16, 10, 8 }; + static const uint8_t div_4000[] = { 20, 12, 10 }; + static const uint8_t div_5333[] = { 24, 16, 14 }; + const uint8_t *div_table; + unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + uint16_t tmp = 0; + + pci_read_config_word(dev->pdev, GCFGC, &tmp); + + cdclk_sel = ((tmp >> 8) & 0x1f) - 1; + + if (cdclk_sel >= ARRAY_SIZE(div_3200)) + goto fail; + + switch (vco) { + case 3200000: + div_table = div_3200; + break; + case 4000000: + div_table = div_4000; + break; + case 5333333: + div_table = div_5333; + break; + default: + goto fail; + } + + return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]); + +fail: + DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n", vco, tmp); + return 200000; +} + +static int g33_get_display_clock_speed(struct drm_device *dev) +{ + static const uint8_t div_3200[] = { 12, 10, 8, 7, 5, 16 }; + static const uint8_t div_4000[] = { 14, 12, 10, 8, 6, 20 }; + static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 }; + static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 }; + const uint8_t *div_table; + unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + uint16_t tmp = 0; + + pci_read_config_word(dev->pdev, GCFGC, &tmp); + + cdclk_sel = (tmp >> 4) & 0x7; + + if (cdclk_sel >= ARRAY_SIZE(div_3200)) + goto fail; + + switch (vco) { + case 3200000: + div_table = div_3200; + break; + case 4000000: + div_table = div_4000; + break; + case 4800000: + div_table = div_4800; + break; + case 5333333: + div_table = div_5333; + break; + default: + goto fail; + } + + return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]); + +fail: + DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n", vco, tmp); + return 190476; +} + static void intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den) { @@ -7064,8 +7337,8 @@ void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n) intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2); } -static void vlv_update_pll(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +static void vlv_compute_dpll(struct intel_crtc *crtc, + struct intel_crtc_state *pipe_config) { u32 dpll, dpll_md; @@ -7074,8 +7347,8 @@ static void vlv_update_pll(struct intel_crtc *crtc, * clock for pipe B, since VGA hotplug / manual detection depends * on it. */ - dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REFA_CLK_ENABLE_VLV | - DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_CLOCK_VLV; + dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REF_CLK_ENABLE_VLV | + DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_REF_CLK_VLV; /* We should never disable this, set it here for state tracking */ if (crtc->pipe == PIPE_B) dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; @@ -7178,11 +7451,11 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, mutex_unlock(&dev_priv->sb_lock); } -static void chv_update_pll(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +static void chv_compute_dpll(struct intel_crtc *crtc, + struct intel_crtc_state *pipe_config) { - pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV | - DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS | + pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV | + DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS | DPLL_VCO_ENABLE; if (crtc->pipe != PIPE_A) pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; @@ -7197,7 +7470,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int pipe = crtc->pipe; - int dpll_reg = DPLL(crtc->pipe); + i915_reg_t dpll_reg = DPLL(crtc->pipe); enum dpio_channel port = vlv_pipe_to_channel(pipe); u32 loopfilter, tribuf_calcntr; u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac; @@ -7238,8 +7511,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc, 1 << DPIO_CHV_N_DIV_SHIFT); /* M2 fraction division */ - if (bestm2_frac) - vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac); + vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac); /* M2 fraction division enable */ dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port)); @@ -7318,11 +7590,11 @@ void vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, }; if (IS_CHERRYVIEW(dev)) { - chv_update_pll(crtc, &pipe_config); + chv_compute_dpll(crtc, &pipe_config); chv_prepare_pll(crtc, &pipe_config); chv_enable_pll(crtc, &pipe_config); } else { - vlv_update_pll(crtc, &pipe_config); + vlv_compute_dpll(crtc, &pipe_config); vlv_prepare_pll(crtc, &pipe_config); vlv_enable_pll(crtc, &pipe_config); } @@ -7344,10 +7616,10 @@ void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe) vlv_disable_pll(to_i915(dev), pipe); } -static void i9xx_update_pll(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state, - intel_clock_t *reduced_clock, - int num_connectors) +static void i9xx_compute_dpll(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + intel_clock_t *reduced_clock, + int num_connectors) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -7421,10 +7693,10 @@ static void i9xx_update_pll(struct intel_crtc *crtc, } } -static void i8xx_update_pll(struct intel_crtc *crtc, - struct intel_crtc_state *crtc_state, - intel_clock_t *reduced_clock, - int num_connectors) +static void i8xx_compute_dpll(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + intel_clock_t *reduced_clock, + int num_connectors) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -7465,8 +7737,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe = intel_crtc->pipe; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; - struct drm_display_mode *adjusted_mode = - &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; uint32_t crtc_vtotal, crtc_vblank_end; int vsyncshift = 0; @@ -7584,9 +7855,14 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode, mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end; mode->flags = pipe_config->base.adjusted_mode.flags; + mode->type = DRM_MODE_TYPE_DRIVER; mode->clock = pipe_config->base.adjusted_mode.crtc_clock; mode->flags |= pipe_config->base.adjusted_mode.flags; + + mode->hsync = drm_mode_hsync(mode); + mode->vrefresh = drm_mode_vrefresh(mode); + drm_mode_set_name(mode); } static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) @@ -7658,10 +7934,8 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int refclk, num_connectors = 0; - intel_clock_t clock, reduced_clock; - bool ok, has_reduced_clock = false; - bool is_lvds = false, is_dsi = false; - struct intel_encoder *encoder; + intel_clock_t clock; + bool ok; const intel_limit_t *limit; struct drm_atomic_state *state = crtc_state->base.state; struct drm_connector *connector; @@ -7671,29 +7945,14 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != &crtc->base) - continue; - - encoder = to_intel_encoder(connector_state->best_encoder); - - switch (encoder->type) { - case INTEL_OUTPUT_LVDS: - is_lvds = true; - break; - case INTEL_OUTPUT_DSI: - is_dsi = true; - break; - default: - break; - } + if (crtc_state->has_dsi_encoder) + return 0; - num_connectors++; + for_each_connector_in_state(state, connector, connector_state, i) { + if (connector_state->crtc == &crtc->base) + num_connectors++; } - if (is_dsi) - return 0; - if (!crtc_state->clock_set) { refclk = i9xx_get_refclk(crtc_state, num_connectors); @@ -7712,19 +7971,6 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - if (is_lvds && dev_priv->lvds_downclock_avail) { - /* - * Ensure we match the reduced clock's P to the target - * clock. If the clocks don't match, we can't switch - * the display clock by using the FP0/FP1. In such case - * we will disable the LVDS downclock feature. - */ - has_reduced_clock = - dev_priv->display.find_dpll(limit, crtc_state, - dev_priv->lvds_downclock, - refclk, &clock, - &reduced_clock); - } /* Compat-code for transition, will disappear. */ crtc_state->dpll.n = clock.n; crtc_state->dpll.m1 = clock.m1; @@ -7734,17 +7980,15 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, } if (IS_GEN2(dev)) { - i8xx_update_pll(crtc, crtc_state, - has_reduced_clock ? &reduced_clock : NULL, - num_connectors); + i8xx_compute_dpll(crtc, crtc_state, NULL, + num_connectors); } else if (IS_CHERRYVIEW(dev)) { - chv_update_pll(crtc, crtc_state); + chv_compute_dpll(crtc, crtc_state); } else if (IS_VALLEYVIEW(dev)) { - vlv_update_pll(crtc, crtc_state); + vlv_compute_dpll(crtc, crtc_state); } else { - i9xx_update_pll(crtc, crtc_state, - has_reduced_clock ? &reduced_clock : NULL, - num_connectors); + i9xx_compute_dpll(crtc, crtc_state, NULL, + num_connectors); } return 0; @@ -7804,10 +8048,7 @@ static void vlv_crtc_clock_get(struct intel_crtc *crtc, clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7; clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f; - vlv_clock(refclk, &clock); - - /* clock.dot is the fast clock */ - pipe_config->port_clock = clock.dot / 5; + pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock); } static void @@ -7906,10 +8147,7 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc, clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7; clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f; - chv_clock(refclk, &clock); - - /* clock.dot is the fast clock */ - pipe_config->port_clock = clock.dot / 5; + pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock); } static bool i9xx_get_pipe_config(struct intel_crtc *crtc, @@ -7999,6 +8237,14 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, else i9xx_crtc_clock_get(crtc, pipe_config); + /* + * Normally the dotclock is filled in by the encoder .get_config() + * but in case the pipe is enabled w/o any ports we need a sane + * default. + */ + pipe_config->base.adjusted_mode.crtc_clock = + pipe_config->port_clock / pipe_config->pixel_multiplier; + return true; } @@ -8260,8 +8506,7 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread, if (WARN(with_fdi && !with_spread, "FDI requires downspread\n")) with_spread = true; - if (WARN(dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE && - with_fdi, "LP PCH doesn't have FDI\n")) + if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n")) with_fdi = false; mutex_lock(&dev_priv->sb_lock); @@ -8284,8 +8529,7 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread, } } - reg = (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) ? - SBI_GEN0 : SBI_DBUFF0; + reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0; tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK); tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE; intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK); @@ -8301,8 +8545,7 @@ static void lpt_disable_clkout_dp(struct drm_device *dev) mutex_lock(&dev_priv->sb_lock); - reg = (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) ? - SBI_GEN0 : SBI_DBUFF0; + reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0; tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK); tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE; intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK); @@ -8558,9 +8801,7 @@ static bool ironlake_compute_clocks(struct drm_crtc *crtc, struct drm_i915_private *dev_priv = dev->dev_private; int refclk; const intel_limit_t *limit; - bool ret, is_lvds = false; - - is_lvds = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS); + bool ret; refclk = ironlake_get_refclk(crtc_state); @@ -8576,20 +8817,6 @@ static bool ironlake_compute_clocks(struct drm_crtc *crtc, if (!ret) return false; - if (is_lvds && dev_priv->lvds_downclock_avail) { - /* - * Ensure we match the reduced clock's P to the target clock. - * If the clocks don't match, we can't switch the display clock - * by using the FP0/FP1. In such case we will disable the LVDS - * downclock feature. - */ - *has_reduced_clock = - dev_priv->display.find_dpll(limit, crtc_state, - dev_priv->lvds_downclock, - refclk, clock, - reduced_clock); - } - return true; } @@ -8718,7 +8945,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc, memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - is_lvds = intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS); + is_lvds = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS); WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)), "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev)); @@ -9147,8 +9374,8 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n"); I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n"); - I915_STATE_WARN(I915_READ(WRPLL_CTL1) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n"); - I915_STATE_WARN(I915_READ(WRPLL_CTL2) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n"); + I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n"); + I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n"); I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n"); I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE, "CPU PWM1 enabled\n"); @@ -9297,6 +9524,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) } intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + intel_update_cdclk(dev_priv->dev); } /* @@ -9329,7 +9557,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv) DRM_DEBUG_KMS("Enabling package C8+\n"); - if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { + if (HAS_PCH_LPT_LP(dev)) { val = I915_READ(SOUTH_DSPCLK_GATE_D); val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; I915_WRITE(SOUTH_DSPCLK_GATE_D, val); @@ -9349,7 +9577,7 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv) hsw_restore_lcpll(dev_priv); lpt_init_pch_refclk(dev); - if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { + if (HAS_PCH_LPT_LP(dev)) { val = I915_READ(SOUTH_DSPCLK_GATE_D); val |= PCH_LP_PARTITION_LEVEL_DISABLE; I915_WRITE(SOUTH_DSPCLK_GATE_D, val); @@ -9358,21 +9586,156 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv) intel_prepare_ddi(dev); } -static void broxton_modeset_global_resources(struct drm_atomic_state *old_state) +static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state) { struct drm_device *dev = old_state->dev; + unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk; + + broxton_set_cdclk(dev, req_cdclk); +} + +/* compute the max rate for new configuration */ +static int ilk_max_pixel_rate(struct drm_atomic_state *state) +{ + struct intel_crtc *intel_crtc; + struct intel_crtc_state *crtc_state; + int max_pixel_rate = 0; + + for_each_intel_crtc(state->dev, intel_crtc) { + int pixel_rate; + + crtc_state = intel_atomic_get_crtc_state(state, intel_crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + if (!crtc_state->base.enable) + continue; + + pixel_rate = ilk_pipe_pixel_rate(crtc_state); + + /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ + if (IS_BROADWELL(state->dev) && crtc_state->ips_enabled) + pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95); + + max_pixel_rate = max(max_pixel_rate, pixel_rate); + } + + return max_pixel_rate; +} + +static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) +{ struct drm_i915_private *dev_priv = dev->dev_private; - int max_pixclk = intel_mode_max_pixclk(dev, NULL); - int req_cdclk; + uint32_t val, data; + int ret; + + if (WARN((I915_READ(LCPLL_CTL) & + (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK | + LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE | + LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW | + LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK, + "trying to change cdclk frequency with cdclk not enabled\n")) + return; + + mutex_lock(&dev_priv->rps.hw_lock); + ret = sandybridge_pcode_write(dev_priv, + BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0); + mutex_unlock(&dev_priv->rps.hw_lock); + if (ret) { + DRM_ERROR("failed to inform pcode about cdclk change\n"); + return; + } + + val = I915_READ(LCPLL_CTL); + val |= LCPLL_CD_SOURCE_FCLK; + I915_WRITE(LCPLL_CTL, val); + + if (wait_for_atomic_us(I915_READ(LCPLL_CTL) & + LCPLL_CD_SOURCE_FCLK_DONE, 1)) + DRM_ERROR("Switching to FCLK failed\n"); + + val = I915_READ(LCPLL_CTL); + val &= ~LCPLL_CLK_FREQ_MASK; - /* see the comment in valleyview_modeset_global_resources */ - if (WARN_ON(max_pixclk < 0)) + switch (cdclk) { + case 450000: + val |= LCPLL_CLK_FREQ_450; + data = 0; + break; + case 540000: + val |= LCPLL_CLK_FREQ_54O_BDW; + data = 1; + break; + case 337500: + val |= LCPLL_CLK_FREQ_337_5_BDW; + data = 2; + break; + case 675000: + val |= LCPLL_CLK_FREQ_675_BDW; + data = 3; + break; + default: + WARN(1, "invalid cdclk frequency\n"); return; + } + + I915_WRITE(LCPLL_CTL, val); + + val = I915_READ(LCPLL_CTL); + val &= ~LCPLL_CD_SOURCE_FCLK; + I915_WRITE(LCPLL_CTL, val); + + if (wait_for_atomic_us((I915_READ(LCPLL_CTL) & + LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1)) + DRM_ERROR("Switching back to LCPLL failed\n"); + + mutex_lock(&dev_priv->rps.hw_lock); + sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data); + mutex_unlock(&dev_priv->rps.hw_lock); - req_cdclk = broxton_calc_cdclk(dev_priv, max_pixclk); + intel_update_cdclk(dev); - if (req_cdclk != dev_priv->cdclk_freq) - broxton_set_cdclk(dev, req_cdclk); + WARN(cdclk != dev_priv->cdclk_freq, + "cdclk requested %d kHz but got %d kHz\n", + cdclk, dev_priv->cdclk_freq); +} + +static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->dev); + int max_pixclk = ilk_max_pixel_rate(state); + int cdclk; + + /* + * FIXME should also account for plane ratio + * once 64bpp pixel formats are supported. + */ + if (max_pixclk > 540000) + cdclk = 675000; + else if (max_pixclk > 450000) + cdclk = 540000; + else if (max_pixclk > 337500) + cdclk = 450000; + else + cdclk = 337500; + + if (cdclk > dev_priv->max_cdclk_freq) { + DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n", + cdclk, dev_priv->max_cdclk_freq); + return -EINVAL; + } + + to_intel_atomic_state(state)->cdclk = cdclk; + + return 0; +} + +static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state) +{ + struct drm_device *dev = old_state->dev; + unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk; + + broadwell_set_cdclk(dev, req_cdclk); } static int haswell_crtc_compute_clock(struct intel_crtc *crtc, @@ -9452,6 +9815,9 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv, case PORT_CLK_SEL_WRPLL2: pipe_config->shared_dpll = DPLL_ID_WRPLL2; break; + case PORT_CLK_SEL_SPLL: + pipe_config->shared_dpll = DPLL_ID_SPLL; + break; } } @@ -9468,7 +9834,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc, port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT; - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) skylake_get_ddi_pll(dev_priv, port, pipe_config); else if (IS_BROXTON(dev)) bxt_get_ddi_pll(dev_priv, port, pipe_config); @@ -9560,12 +9926,10 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, } if (intel_display_power_is_enabled(dev_priv, pfit_domain)) { - if (INTEL_INFO(dev)->gen == 9) + if (INTEL_INFO(dev)->gen >= 9) skylake_get_pfit_config(crtc, pipe_config); - else if (INTEL_INFO(dev)->gen < 9) - ironlake_get_pfit_config(crtc, pipe_config); else - MISSING_CASE(INTEL_INFO(dev)->gen); + ironlake_get_pfit_config(crtc, pipe_config); } if (IS_HASWELL(dev)) @@ -9622,13 +9986,13 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base) /* On these chipsets we can only modify the base/size/stride * whilst the cursor is disabled. */ - I915_WRITE(_CURACNTR, 0); - POSTING_READ(_CURACNTR); + I915_WRITE(CURCNTR(PIPE_A), 0); + POSTING_READ(CURCNTR(PIPE_A)); intel_crtc->cursor_cntl = 0; } if (intel_crtc->cursor_base != base) { - I915_WRITE(_CURABASE, base); + I915_WRITE(CURBASE(PIPE_A), base); intel_crtc->cursor_base = base; } @@ -9638,8 +10002,8 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base) } if (intel_crtc->cursor_cntl != cntl) { - I915_WRITE(_CURACNTR, cntl); - POSTING_READ(_CURACNTR); + I915_WRITE(CURCNTR(PIPE_A), cntl); + POSTING_READ(CURCNTR(PIPE_A)); intel_crtc->cursor_cntl = cntl; } } @@ -9671,7 +10035,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base) } cntl |= pipe << 28; /* Connect to correct pipe */ - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + if (HAS_DDI(dev)) cntl |= CURSOR_PIPE_CSC_ENABLE; } @@ -9699,8 +10063,9 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - int x = crtc->cursor_x; - int y = crtc->cursor_y; + struct drm_plane_state *cursor_state = crtc->cursor->state; + int x = cursor_state->crtc_x; + int y = cursor_state->crtc_y; u32 base = 0, pos = 0; if (on) @@ -9713,7 +10078,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, base = 0; if (x < 0) { - if (x + intel_crtc->base.cursor->state->crtc_w <= 0) + if (x + cursor_state->crtc_w <= 0) base = 0; pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT; @@ -9722,7 +10087,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, pos |= x << CURSOR_X_SHIFT; if (y < 0) { - if (y + intel_crtc->base.cursor->state->crtc_h <= 0) + if (y + cursor_state->crtc_h <= 0) base = 0; pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT; @@ -9738,8 +10103,8 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, /* ILK+ do this automagically */ if (HAS_GMCH_DISPLAY(dev) && crtc->cursor->state->rotation == BIT(DRM_ROTATE_180)) { - base += (intel_crtc->base.cursor->state->crtc_h * - intel_crtc->base.cursor->state->crtc_w - 1) * 4; + base += (cursor_state->crtc_h * + cursor_state->crtc_w - 1) * 4; } if (IS_845G(dev) || IS_I865G(dev)) @@ -9815,20 +10180,17 @@ __intel_framebuffer_create(struct drm_device *dev, int ret; intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); - if (!intel_fb) { - drm_gem_object_unreference(&obj->base); + if (!intel_fb) return ERR_PTR(-ENOMEM); - } ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj); if (ret) goto err; return &intel_fb->base; + err: - drm_gem_object_unreference(&obj->base); kfree(intel_fb); - return ERR_PTR(ret); } @@ -9868,6 +10230,7 @@ intel_framebuffer_create_for_mode(struct drm_device *dev, struct drm_display_mode *mode, int depth, int bpp) { + struct drm_framebuffer *fb; struct drm_i915_gem_object *obj; struct drm_mode_fb_cmd2 mode_cmd = { 0 }; @@ -9882,14 +10245,18 @@ intel_framebuffer_create_for_mode(struct drm_device *dev, bpp); mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth); - return intel_framebuffer_create(dev, &mode_cmd, obj); + fb = intel_framebuffer_create(dev, &mode_cmd, obj); + if (IS_ERR(fb)) + drm_gem_object_unreference_unlocked(&obj->base); + + return fb; } static struct drm_framebuffer * mode_fits_in_fbdev(struct drm_device *dev, struct drm_display_mode *mode) { -#ifdef CONFIG_DRM_I915_FBDEV +#ifdef CONFIG_DRM_FBDEV_EMULATION struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; struct drm_framebuffer *fb; @@ -9978,7 +10345,7 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, retry: ret = drm_modeset_lock(&config->connection_mutex, ctx); if (ret) - goto fail_unlock; + goto fail; /* * Algorithm gets a little messy: @@ -9996,10 +10363,10 @@ retry: ret = drm_modeset_lock(&crtc->mutex, ctx); if (ret) - goto fail_unlock; + goto fail; ret = drm_modeset_lock(&crtc->primary->mutex, ctx); if (ret) - goto fail_unlock; + goto fail; old->dpms_mode = connector->dpms; old->load_detect_temp = false; @@ -10018,9 +10385,6 @@ retry: continue; if (possible_crtc->state->enable) continue; - /* This can occur when applying the pipe A quirk on resume. */ - if (to_intel_crtc(possible_crtc)->new_enabled) - continue; crtc = possible_crtc; break; @@ -10031,20 +10395,17 @@ retry: */ if (!crtc) { DRM_DEBUG_KMS("no pipe available for load-detect\n"); - goto fail_unlock; + goto fail; } ret = drm_modeset_lock(&crtc->mutex, ctx); if (ret) - goto fail_unlock; + goto fail; ret = drm_modeset_lock(&crtc->primary->mutex, ctx); if (ret) - goto fail_unlock; - intel_encoder->new_crtc = to_intel_crtc(crtc); - to_intel_connector(connector)->new_encoder = intel_encoder; + goto fail; intel_crtc = to_intel_crtc(crtc); - intel_crtc->new_enabled = true; old->dpms_mode = connector->dpms; old->load_detect_temp = true; old->release_fb = NULL; @@ -10100,7 +10461,7 @@ retry: drm_mode_copy(&crtc_state->base.mode, mode); - if (intel_set_mode(crtc, state, true)) { + if (drm_atomic_commit(state)) { DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n"); if (old->release_fb) old->release_fb->funcs->destroy(old->release_fb); @@ -10112,9 +10473,7 @@ retry: intel_wait_for_vblank(dev, intel_crtc->pipe); return true; - fail: - intel_crtc->new_enabled = crtc->state->enable; -fail_unlock: +fail: drm_atomic_state_free(state); state = NULL; @@ -10160,10 +10519,6 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, if (IS_ERR(crtc_state)) goto fail; - to_intel_connector(connector)->new_encoder = NULL; - intel_encoder->new_crtc = NULL; - intel_crtc->new_enabled = false; - connector_state->best_encoder = NULL; connector_state->crtc = NULL; @@ -10174,7 +10529,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, if (ret) goto fail; - ret = intel_set_mode(crtc, state, true); + ret = drm_atomic_commit(state); if (ret) goto fail; @@ -10222,6 +10577,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, u32 dpll = pipe_config->dpll_hw_state.dpll; u32 fp; intel_clock_t clock; + int port_clock; int refclk = i9xx_pll_refclk(dev, pipe_config); if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0) @@ -10262,9 +10618,9 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, } if (IS_PINEVIEW(dev)) - pineview_clock(refclk, &clock); + port_clock = pnv_calc_dpll_params(refclk, &clock); else - i9xx_clock(refclk, &clock); + port_clock = i9xx_calc_dpll_params(refclk, &clock); } else { u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS); bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN); @@ -10290,7 +10646,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, clock.p2 = 2; } - i9xx_clock(refclk, &clock); + port_clock = i9xx_calc_dpll_params(refclk, &clock); } /* @@ -10298,7 +10654,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, * port_clock to compute adjusted_mode.crtc_clock in the * encoder's get_config() function. */ - pipe_config->port_clock = clock.dot; + pipe_config->port_clock = port_clock; } int intel_dotclock_calculate(int link_freq, @@ -10387,42 +10743,6 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev, return mode; } -static void intel_decrease_pllclock(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - - if (!HAS_GMCH_DISPLAY(dev)) - return; - - if (!dev_priv->lvds_downclock_avail) - return; - - /* - * Since this is called by a timer, we should never get here in - * the manual case. - */ - if (!HAS_PIPE_CXSR(dev) && intel_crtc->lowfreq_avail) { - int pipe = intel_crtc->pipe; - int dpll_reg = DPLL(pipe); - int dpll; - - DRM_DEBUG_DRIVER("downclocking LVDS\n"); - - assert_panel_unlocked(dev_priv, pipe); - - dpll = I915_READ(dpll_reg); - dpll |= DISPLAY_RATE_SELECT_FPA1; - I915_WRITE(dpll_reg, dpll); - intel_wait_for_vblank(dev, pipe); - dpll = I915_READ(dpll_reg); - if (!(dpll & DISPLAY_RATE_SELECT_FPA1)) - DRM_DEBUG_DRIVER("failed to downclock LVDS!\n"); - } - -} - void intel_mark_busy(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -10440,20 +10760,12 @@ void intel_mark_busy(struct drm_device *dev) void intel_mark_idle(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_crtc *crtc; if (!dev_priv->mm.busy) return; dev_priv->mm.busy = false; - for_each_crtc(dev, crtc) { - if (!crtc->primary->fb) - continue; - - intel_decrease_pllclock(crtc); - } - if (INTEL_INFO(dev)->gen >= 6) gen6_rps_idle(dev->dev_private); @@ -10485,24 +10797,23 @@ static void intel_unpin_work_fn(struct work_struct *__work) { struct intel_unpin_work *work = container_of(__work, struct intel_unpin_work, work); - struct drm_device *dev = work->crtc->dev; - enum pipe pipe = to_intel_crtc(work->crtc)->pipe; + struct intel_crtc *crtc = to_intel_crtc(work->crtc); + struct drm_device *dev = crtc->base.dev; + struct drm_plane *primary = crtc->base.primary; mutex_lock(&dev->struct_mutex); - intel_unpin_fb_obj(work->old_fb, work->crtc->primary->state); + intel_unpin_fb_obj(work->old_fb, primary->state); drm_gem_object_unreference(&work->pending_flip_obj->base); - intel_fbc_update(dev); - if (work->flip_queued_req) i915_gem_request_assign(&work->flip_queued_req, NULL); mutex_unlock(&dev->struct_mutex); - intel_frontbuffer_flip_complete(dev, INTEL_FRONTBUFFER_PRIMARY(pipe)); + intel_frontbuffer_flip_complete(dev, to_intel_plane(primary)->frontbuffer_bit); drm_framebuffer_unreference(work->old_fb); - BUG_ON(atomic_read(&to_intel_crtc(work->crtc)->unpin_work_count) == 0); - atomic_dec(&to_intel_crtc(work->crtc)->unpin_work_count); + BUG_ON(atomic_read(&crtc->unpin_work_count) == 0); + atomic_dec(&crtc->unpin_work_count); kfree(work); } @@ -10596,7 +10907,7 @@ static bool page_flip_finished(struct intel_crtc *crtc) */ return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) == crtc->unpin_work->gtt_offset && - g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_GM45(crtc->pipe)), + g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)), crtc->unpin_work->flip_count); } @@ -10622,11 +10933,11 @@ void intel_prepare_page_flip(struct drm_device *dev, int plane) spin_unlock_irqrestore(&dev->event_lock, flags); } -static inline void intel_mark_page_flip_active(struct intel_crtc *intel_crtc) +static inline void intel_mark_page_flip_active(struct intel_unpin_work *work) { /* Ensure that the work item is consistent when activating it ... */ smp_wmb(); - atomic_set(&intel_crtc->unpin_work->pending, INTEL_FLIP_PENDING); + atomic_set(&work->pending, INTEL_FLIP_PENDING); /* and that it is marked active as soon as the irq could fire. */ smp_wmb(); } @@ -10635,14 +10946,15 @@ static int intel_gen2_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags) { + struct intel_engine_cs *ring = req->ring; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 flip_mask; int ret; - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; @@ -10661,8 +10973,7 @@ static int intel_gen2_queue_flip(struct drm_device *dev, intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); intel_ring_emit(ring, 0); /* aux display base address, unused */ - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); + intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -10670,14 +10981,15 @@ static int intel_gen3_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags) { + struct intel_engine_cs *ring = req->ring; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 flip_mask; int ret; - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; @@ -10693,8 +11005,7 @@ static int intel_gen3_queue_flip(struct drm_device *dev, intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); intel_ring_emit(ring, MI_NOOP); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); + intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -10702,15 +11013,16 @@ static int intel_gen4_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags) { + struct intel_engine_cs *ring = req->ring; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pf, pipesrc; int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -10732,8 +11044,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev, pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; intel_ring_emit(ring, pf | pipesrc); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); + intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -10741,15 +11052,16 @@ static int intel_gen6_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags) { + struct intel_engine_cs *ring = req->ring; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pf, pipesrc; int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -10768,8 +11080,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev, pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; intel_ring_emit(ring, pf | pipesrc); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); + intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -10777,9 +11088,10 @@ static int intel_gen7_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags) { + struct intel_engine_cs *ring = req->ring; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t plane_bit = 0; int len, ret; @@ -10821,11 +11133,11 @@ static int intel_gen7_queue_flip(struct drm_device *dev, * then do the cacheline alignment, and finally emit the * MI_DISPLAY_FLIP. */ - ret = intel_ring_cacheline_align(ring); + ret = intel_ring_cacheline_align(req); if (ret) return ret; - ret = intel_ring_begin(ring, len); + ret = intel_ring_begin(req, len); if (ret) return ret; @@ -10840,17 +11152,17 @@ static int intel_gen7_queue_flip(struct drm_device *dev, */ if (ring->id == RCS) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, DERRMR); + intel_ring_emit_reg(ring, DERRMR); intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE | DERRMR_PIPEB_PRI_FLIP_DONE | DERRMR_PIPEC_PRI_FLIP_DONE)); if (IS_GEN8(dev)) - intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8(1) | + intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT); else - intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | + intel_ring_emit(ring, MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT); - intel_ring_emit(ring, DERRMR); + intel_ring_emit_reg(ring, DERRMR); intel_ring_emit(ring, ring->scratch.gtt_offset + 256); if (IS_GEN8(dev)) { intel_ring_emit(ring, 0); @@ -10863,8 +11175,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev, intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); intel_ring_emit(ring, (MI_NOOP)); - intel_mark_page_flip_active(intel_crtc); - __intel_ring_advance(ring); + intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -10891,17 +11202,23 @@ static bool use_mmio_flip(struct intel_engine_cs *ring, return true; else if (i915.enable_execlists) return true; + else if (obj->base.dma_buf && + !reservation_object_test_signaled_rcu(obj->base.dma_buf->resv, + false)) + return true; else return ring != i915_gem_request_get_ring(obj->last_write_req); } -static void skl_do_mmio_flip(struct intel_crtc *intel_crtc) +static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, + unsigned int rotation, + struct intel_unpin_work *work) { struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_framebuffer *fb = intel_crtc->base.primary->fb; const enum pipe pipe = intel_crtc->pipe; - u32 ctl, stride; + u32 ctl, stride, tile_height; ctl = I915_READ(PLANE_CTL(pipe, 0)); ctl &= ~PLANE_CTL_TILED_MASK; @@ -10925,9 +11242,16 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc) * The stride is either expressed as a multiple of 64 bytes chunks for * linear buffers or in number of tiles for tiled buffers. */ - stride = fb->pitches[0] / - intel_fb_stride_alignment(dev, fb->modifier[0], - fb->pixel_format); + if (intel_rotation_90_or_270(rotation)) { + /* stride = Surface height in tiles */ + tile_height = intel_tile_height(dev, fb->pixel_format, + fb->modifier[0], 0); + stride = DIV_ROUND_UP(fb->height, tile_height); + } else { + stride = fb->pitches[0] / + intel_fb_stride_alignment(dev, fb->modifier[0], + fb->pixel_format); + } /* * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on @@ -10936,21 +11260,21 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc) I915_WRITE(PLANE_CTL(pipe, 0), ctl); I915_WRITE(PLANE_STRIDE(pipe, 0), stride); - I915_WRITE(PLANE_SURF(pipe, 0), intel_crtc->unpin_work->gtt_offset); + I915_WRITE(PLANE_SURF(pipe, 0), work->gtt_offset); POSTING_READ(PLANE_SURF(pipe, 0)); } -static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc) +static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc, + struct intel_unpin_work *work) { struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_framebuffer *intel_fb = to_intel_framebuffer(intel_crtc->base.primary->fb); struct drm_i915_gem_object *obj = intel_fb->obj; + i915_reg_t reg = DSPCNTR(intel_crtc->plane); u32 dspcntr; - u32 reg; - reg = DSPCNTR(intel_crtc->plane); dspcntr = I915_READ(reg); if (obj->tiling_mode != I915_TILING_NONE) @@ -10960,59 +11284,67 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc) I915_WRITE(reg, dspcntr); - I915_WRITE(DSPSURF(intel_crtc->plane), - intel_crtc->unpin_work->gtt_offset); + I915_WRITE(DSPSURF(intel_crtc->plane), work->gtt_offset); POSTING_READ(DSPSURF(intel_crtc->plane)); - } /* * XXX: This is the temporary way to update the plane registers until we get * around to using the usual plane update functions for MMIO flips */ -static void intel_do_mmio_flip(struct intel_crtc *intel_crtc) +static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip) { - struct drm_device *dev = intel_crtc->base.dev; - bool atomic_update; - u32 start_vbl_count; + struct intel_crtc *crtc = mmio_flip->crtc; + struct intel_unpin_work *work; + + spin_lock_irq(&crtc->base.dev->event_lock); + work = crtc->unpin_work; + spin_unlock_irq(&crtc->base.dev->event_lock); + if (work == NULL) + return; - intel_mark_page_flip_active(intel_crtc); + intel_mark_page_flip_active(work); - atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + intel_pipe_update_start(crtc); - if (INTEL_INFO(dev)->gen >= 9) - skl_do_mmio_flip(intel_crtc); + if (INTEL_INFO(mmio_flip->i915)->gen >= 9) + skl_do_mmio_flip(crtc, mmio_flip->rotation, work); else /* use_mmio_flip() retricts MMIO flips to ilk+ */ - ilk_do_mmio_flip(intel_crtc); + ilk_do_mmio_flip(crtc, work); - if (atomic_update) - intel_pipe_update_end(intel_crtc, start_vbl_count); + intel_pipe_update_end(crtc); } static void intel_mmio_flip_work_func(struct work_struct *work) { struct intel_mmio_flip *mmio_flip = container_of(work, struct intel_mmio_flip, work); + struct intel_framebuffer *intel_fb = + to_intel_framebuffer(mmio_flip->crtc->base.primary->fb); + struct drm_i915_gem_object *obj = intel_fb->obj; - if (mmio_flip->req) + if (mmio_flip->req) { WARN_ON(__i915_wait_request(mmio_flip->req, mmio_flip->crtc->reset_counter, false, NULL, &mmio_flip->i915->rps.mmioflips)); + i915_gem_request_unreference__unlocked(mmio_flip->req); + } - intel_do_mmio_flip(mmio_flip->crtc); + /* For framebuffer backed by dmabuf, wait for fence */ + if (obj->base.dma_buf) + WARN_ON(reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv, + false, false, + MAX_SCHEDULE_TIMEOUT) < 0); - i915_gem_request_unreference__unlocked(mmio_flip->req); + intel_do_mmio_flip(mmio_flip); kfree(mmio_flip); } static int intel_queue_mmio_flip(struct drm_device *dev, struct drm_crtc *crtc, - struct drm_framebuffer *fb, - struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, - uint32_t flags) + struct drm_i915_gem_object *obj) { struct intel_mmio_flip *mmio_flip; @@ -11023,6 +11355,7 @@ static int intel_queue_mmio_flip(struct drm_device *dev, mmio_flip->i915 = to_i915(dev); mmio_flip->req = i915_gem_request_reference(obj->last_write_req); mmio_flip->crtc = to_intel_crtc(crtc); + mmio_flip->rotation = crtc->primary->state->rotation; INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func); schedule_work(&mmio_flip->work); @@ -11034,7 +11367,7 @@ static int intel_default_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, + struct drm_i915_gem_request *req, uint32_t flags) { return -ENODEV; @@ -11051,6 +11384,9 @@ static bool __intel_pageflip_stall_check(struct drm_device *dev, if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE) return true; + if (atomic_read(&work->pending) < INTEL_FLIP_PENDING) + return false; + if (!work->enable_stall_check) return false; @@ -11120,6 +11456,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, struct intel_unpin_work *work; struct intel_engine_cs *ring; bool mmio_flip; + struct drm_i915_gem_request *request = NULL; int ret; /* @@ -11200,7 +11537,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) - work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(pipe)) + 1; + work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1; if (IS_VALLEYVIEW(dev)) { ring = &dev_priv->ring[BCS]; @@ -11224,49 +11561,57 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, * synchronisation, so all we want here is to pin the framebuffer * into the display plane and skip any waits. */ + if (!mmio_flip) { + ret = i915_gem_object_sync(obj, ring, &request); + if (ret) + goto cleanup_pending; + } + ret = intel_pin_and_fence_fb_obj(crtc->primary, fb, - crtc->primary->state, - mmio_flip ? i915_gem_request_get_ring(obj->last_write_req) : ring); + crtc->primary->state); if (ret) goto cleanup_pending; - work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary), obj) - + intel_crtc->dspaddr_offset; + work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary), + obj, 0); + work->gtt_offset += intel_crtc->dspaddr_offset; if (mmio_flip) { - ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring, - page_flip_flags); + ret = intel_queue_mmio_flip(dev, crtc, obj); if (ret) goto cleanup_unpin; i915_gem_request_assign(&work->flip_queued_req, obj->last_write_req); } else { - if (obj->last_write_req) { - ret = i915_gem_check_olr(obj->last_write_req); + if (!request) { + ret = i915_gem_request_alloc(ring, ring->default_context, &request); if (ret) goto cleanup_unpin; } - ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring, + ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request, page_flip_flags); if (ret) goto cleanup_unpin; - i915_gem_request_assign(&work->flip_queued_req, - intel_ring_get_request(ring)); + i915_gem_request_assign(&work->flip_queued_req, request); } + if (request) + i915_add_request_no_flush(request); + work->flip_queued_vblank = drm_crtc_vblank_count(crtc); work->enable_stall_check = true; i915_gem_track_fb(intel_fb_obj(work->old_fb), obj, - INTEL_FRONTBUFFER_PRIMARY(pipe)); - - intel_fbc_disable(dev); - intel_frontbuffer_flip_prepare(dev, INTEL_FRONTBUFFER_PRIMARY(pipe)); + to_intel_plane(primary)->frontbuffer_bit); mutex_unlock(&dev->struct_mutex); + intel_fbc_deactivate(intel_crtc); + intel_frontbuffer_flip_prepare(dev, + to_intel_plane(primary)->frontbuffer_bit); + trace_i915_flip_request(intel_crtc->plane, obj); return 0; @@ -11274,6 +11619,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, cleanup_unpin: intel_unpin_fb_obj(fb, crtc->primary->state); cleanup_pending: + if (request) + i915_gem_request_cancel(request); atomic_dec(&intel_crtc->unpin_work_count); mutex_unlock(&dev->struct_mutex); cleanup: @@ -11292,8 +11639,35 @@ free_work: kfree(work); if (ret == -EIO) { + struct drm_atomic_state *state; + struct drm_plane_state *plane_state; + out_hang: - ret = intel_plane_restore(primary); + state = drm_atomic_state_alloc(dev); + if (!state) + return -ENOMEM; + state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc); + +retry: + plane_state = drm_atomic_get_plane_state(state, primary); + ret = PTR_ERR_OR_ZERO(plane_state); + if (!ret) { + drm_atomic_set_fb_for_plane(plane_state, fb); + + ret = drm_atomic_set_crtc_for_plane(plane_state, crtc); + if (!ret) + ret = drm_atomic_commit(state); + } + + if (ret == -EDEADLK) { + drm_modeset_backoff(state->acquire_ctx); + drm_atomic_state_clear(state); + goto retry; + } + + if (ret) + drm_atomic_state_free(state); + if (ret == 0 && event) { spin_lock_irq(&dev->event_lock); drm_send_vblank_event(dev, pipe, event); @@ -11303,44 +11677,296 @@ out_hang: return ret; } -static const struct drm_crtc_helper_funcs intel_helper_funcs = { - .mode_set_base_atomic = intel_pipe_set_base_atomic, - .load_lut = intel_crtc_load_lut, - .atomic_begin = intel_begin_crtc_commit, - .atomic_flush = intel_finish_crtc_commit, -}; /** - * intel_modeset_update_staged_output_state + * intel_wm_need_update - Check whether watermarks need updating + * @plane: drm plane + * @state: new plane state * - * Updates the staged output configuration state, e.g. after we've read out the - * current hw state. + * Check current plane state versus the new one to determine whether + * watermarks need to be recalculated. + * + * Returns true or false. */ -static void intel_modeset_update_staged_output_state(struct drm_device *dev) +static bool intel_wm_need_update(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct intel_plane_state *new = to_intel_plane_state(state); + struct intel_plane_state *cur = to_intel_plane_state(plane->state); + + /* Update watermarks on tiling or size changes. */ + if (!plane->state->fb || !state->fb || + plane->state->fb->modifier[0] != state->fb->modifier[0] || + plane->state->rotation != state->rotation || + drm_rect_width(&new->src) != drm_rect_width(&cur->src) || + drm_rect_height(&new->src) != drm_rect_height(&cur->src) || + drm_rect_width(&new->dst) != drm_rect_width(&cur->dst) || + drm_rect_height(&new->dst) != drm_rect_height(&cur->dst)) + return true; + + return false; +} + +static bool needs_scaling(struct intel_plane_state *state) +{ + int src_w = drm_rect_width(&state->src) >> 16; + int src_h = drm_rect_height(&state->src) >> 16; + int dst_w = drm_rect_width(&state->dst); + int dst_h = drm_rect_height(&state->dst); + + return (src_w != dst_w || src_h != dst_h); +} + +int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state, + struct drm_plane_state *plane_state) +{ + struct drm_crtc *crtc = crtc_state->crtc; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct drm_plane *plane = plane_state->plane; + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_plane_state *old_plane_state = + to_intel_plane_state(plane->state); + int idx = intel_crtc->base.base.id, ret; + int i = drm_plane_index(plane); + bool mode_changed = needs_modeset(crtc_state); + bool was_crtc_enabled = crtc->state->active; + bool is_crtc_enabled = crtc_state->active; + bool turn_off, turn_on, visible, was_visible; + struct drm_framebuffer *fb = plane_state->fb; + + if (crtc_state && INTEL_INFO(dev)->gen >= 9 && + plane->type != DRM_PLANE_TYPE_CURSOR) { + ret = skl_update_scaler_plane( + to_intel_crtc_state(crtc_state), + to_intel_plane_state(plane_state)); + if (ret) + return ret; + } + + was_visible = old_plane_state->visible; + visible = to_intel_plane_state(plane_state)->visible; + + if (!was_crtc_enabled && WARN_ON(was_visible)) + was_visible = false; + + if (!is_crtc_enabled && WARN_ON(visible)) + visible = false; + + if (!was_visible && !visible) + return 0; + + turn_off = was_visible && (!visible || mode_changed); + turn_on = visible && (!was_visible || mode_changed); + + DRM_DEBUG_ATOMIC("[CRTC:%i] has [PLANE:%i] with fb %i\n", idx, + plane->base.id, fb ? fb->base.id : -1); + + DRM_DEBUG_ATOMIC("[PLANE:%i] visible %i -> %i, off %i, on %i, ms %i\n", + plane->base.id, was_visible, visible, + turn_off, turn_on, mode_changed); + + if (turn_on) { + intel_crtc->atomic.update_wm_pre = true; + /* must disable cxsr around plane enable/disable */ + if (plane->type != DRM_PLANE_TYPE_CURSOR) { + intel_crtc->atomic.disable_cxsr = true; + /* to potentially re-enable cxsr */ + intel_crtc->atomic.wait_vblank = true; + intel_crtc->atomic.update_wm_post = true; + } + } else if (turn_off) { + intel_crtc->atomic.update_wm_post = true; + /* must disable cxsr around plane enable/disable */ + if (plane->type != DRM_PLANE_TYPE_CURSOR) { + if (is_crtc_enabled) + intel_crtc->atomic.wait_vblank = true; + intel_crtc->atomic.disable_cxsr = true; + } + } else if (intel_wm_need_update(plane, plane_state)) { + intel_crtc->atomic.update_wm_pre = true; + } + + if (visible || was_visible) + intel_crtc->atomic.fb_bits |= + to_intel_plane(plane)->frontbuffer_bit; + + switch (plane->type) { + case DRM_PLANE_TYPE_PRIMARY: + intel_crtc->atomic.pre_disable_primary = turn_off; + intel_crtc->atomic.post_enable_primary = turn_on; + + if (turn_off) { + /* + * FIXME: Actually if we will still have any other + * plane enabled on the pipe we could let IPS enabled + * still, but for now lets consider that when we make + * primary invisible by setting DSPCNTR to 0 on + * update_primary_plane function IPS needs to be + * disable. + */ + intel_crtc->atomic.disable_ips = true; + + intel_crtc->atomic.disable_fbc = true; + } + + /* + * FBC does not work on some platforms for rotated + * planes, so disable it when rotation is not 0 and + * update it when rotation is set back to 0. + * + * FIXME: This is redundant with the fbc update done in + * the primary plane enable function except that that + * one is done too late. We eventually need to unify + * this. + */ + + if (visible && + INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && + dev_priv->fbc.crtc == intel_crtc && + plane_state->rotation != BIT(DRM_ROTATE_0)) + intel_crtc->atomic.disable_fbc = true; + + /* + * BDW signals flip done immediately if the plane + * is disabled, even if the plane enable is already + * armed to occur at the next vblank :( + */ + if (turn_on && IS_BROADWELL(dev)) + intel_crtc->atomic.wait_vblank = true; + + intel_crtc->atomic.update_fbc |= visible || mode_changed; + break; + case DRM_PLANE_TYPE_CURSOR: + break; + case DRM_PLANE_TYPE_OVERLAY: + /* + * WaCxSRDisabledForSpriteScaling:ivb + * + * cstate->update_wm was already set above, so this flag will + * take effect when we commit and program watermarks. + */ + if (IS_IVYBRIDGE(dev) && + needs_scaling(to_intel_plane_state(plane_state)) && + !needs_scaling(old_plane_state)) { + to_intel_crtc_state(crtc_state)->disable_lp_wm = true; + } else if (turn_off && !mode_changed) { + intel_crtc->atomic.wait_vblank = true; + intel_crtc->atomic.update_sprite_watermarks |= + 1 << i; + } + + break; + } + return 0; +} + +static bool encoders_cloneable(const struct intel_encoder *a, + const struct intel_encoder *b) +{ + /* masks could be asymmetric, so check both ways */ + return a == b || (a->cloneable & (1 << b->type) && + b->cloneable & (1 << a->type)); +} + +static bool check_single_encoder_cloning(struct drm_atomic_state *state, + struct intel_crtc *crtc, + struct intel_encoder *encoder) +{ + struct intel_encoder *source_encoder; + struct drm_connector *connector; + struct drm_connector_state *connector_state; + int i; + + for_each_connector_in_state(state, connector, connector_state, i) { + if (connector_state->crtc != &crtc->base) + continue; + + source_encoder = + to_intel_encoder(connector_state->best_encoder); + if (!encoders_cloneable(encoder, source_encoder)) + return false; + } + + return true; +} + +static bool check_encoder_cloning(struct drm_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_crtc *crtc; struct intel_encoder *encoder; - struct intel_connector *connector; + struct drm_connector *connector; + struct drm_connector_state *connector_state; + int i; - for_each_intel_connector(dev, connector) { - connector->new_encoder = - to_intel_encoder(connector->base.encoder); + for_each_connector_in_state(state, connector, connector_state, i) { + if (connector_state->crtc != &crtc->base) + continue; + + encoder = to_intel_encoder(connector_state->best_encoder); + if (!check_single_encoder_cloning(state, crtc, encoder)) + return false; } - for_each_intel_encoder(dev, encoder) { - encoder->new_crtc = - to_intel_crtc(encoder->base.crtc); + return true; +} + +static int intel_crtc_atomic_check(struct drm_crtc *crtc, + struct drm_crtc_state *crtc_state) +{ + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc_state *pipe_config = + to_intel_crtc_state(crtc_state); + struct drm_atomic_state *state = crtc_state->state; + int ret; + bool mode_changed = needs_modeset(crtc_state); + + if (mode_changed && !check_encoder_cloning(state, intel_crtc)) { + DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); + return -EINVAL; } - for_each_intel_crtc(dev, crtc) { - crtc->new_enabled = crtc->base.state->enable; + if (mode_changed && !crtc_state->active) + intel_crtc->atomic.update_wm_post = true; + + if (mode_changed && crtc_state->enable && + dev_priv->display.crtc_compute_clock && + !WARN_ON(pipe_config->shared_dpll != DPLL_ID_PRIVATE)) { + ret = dev_priv->display.crtc_compute_clock(intel_crtc, + pipe_config); + if (ret) + return ret; + } + + ret = 0; + if (dev_priv->display.compute_pipe_wm) { + ret = dev_priv->display.compute_pipe_wm(intel_crtc, state); + if (ret) + return ret; } + + if (INTEL_INFO(dev)->gen >= 9) { + if (mode_changed) + ret = skl_update_scaler_crtc(pipe_config); + + if (!ret) + ret = intel_atomic_setup_scalers(dev, intel_crtc, + pipe_config); + } + + return ret; } -/* Transitional helper to copy current connector/encoder state to - * connector->state. This is needed so that code that is partially - * converted to atomic does the right thing. - */ +static const struct drm_crtc_helper_funcs intel_helper_funcs = { + .mode_set_base_atomic = intel_pipe_set_base_atomic, + .load_lut = intel_crtc_load_lut, + .atomic_begin = intel_begin_crtc_commit, + .atomic_flush = intel_finish_crtc_commit, + .atomic_check = intel_crtc_atomic_check, +}; + static void intel_modeset_update_connector_atomic_state(struct drm_device *dev) { struct intel_connector *connector; @@ -11358,39 +11984,6 @@ static void intel_modeset_update_connector_atomic_state(struct drm_device *dev) } } -/* Fixup legacy state after an atomic state swap. - */ -static void intel_modeset_fixup_state(struct drm_atomic_state *state) -{ - struct intel_crtc *crtc; - struct intel_encoder *encoder; - struct intel_connector *connector; - - for_each_intel_connector(state->dev, connector) { - connector->base.encoder = connector->base.state->best_encoder; - if (connector->base.encoder) - connector->base.encoder->crtc = - connector->base.state->crtc; - } - - /* Update crtc of disabled encoders */ - for_each_intel_encoder(state->dev, encoder) { - int num_connectors = 0; - - for_each_intel_connector(state->dev, connector) - if (connector->base.encoder == &encoder->base) - num_connectors++; - - if (num_connectors == 0) - encoder->base.crtc = NULL; - } - - for_each_intel_crtc(state->dev, crtc) { - crtc->base.enabled = crtc->base.state->enable; - crtc->config = to_intel_crtc_state(crtc->base.state); - } -} - static void connected_sink_compute_bpp(struct intel_connector *connector, struct intel_crtc_state *pipe_config) @@ -11484,14 +12077,16 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n, pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n, pipe_config->fdi_m_n.tu); - DRM_DEBUG_KMS("dp: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", + DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", pipe_config->has_dp_encoder, + pipe_config->lane_count, pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n, pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n, pipe_config->dp_m_n.tu); - DRM_DEBUG_KMS("dp: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n", + DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n", pipe_config->has_dp_encoder, + pipe_config->lane_count, pipe_config->dp_m2_n2.gmch_m, pipe_config->dp_m2_n2.gmch_n, pipe_config->dp_m2_n2.link_m, @@ -11526,19 +12121,22 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide); if (IS_BROXTON(dev)) { - DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, " + DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x," "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, " - "pll6: 0x%x, pll8: 0x%x, pcsdw12: 0x%x\n", + "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n", pipe_config->ddi_pll_sel, pipe_config->dpll_hw_state.ebb0, + pipe_config->dpll_hw_state.ebb4, pipe_config->dpll_hw_state.pll0, pipe_config->dpll_hw_state.pll1, pipe_config->dpll_hw_state.pll2, pipe_config->dpll_hw_state.pll3, pipe_config->dpll_hw_state.pll6, pipe_config->dpll_hw_state.pll8, + pipe_config->dpll_hw_state.pll9, + pipe_config->dpll_hw_state.pll10, pipe_config->dpll_hw_state.pcsdw12); - } else if (IS_SKYLAKE(dev)) { + } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: " "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n", pipe_config->ddi_pll_sel, @@ -11546,9 +12144,10 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, pipe_config->dpll_hw_state.cfgcr1, pipe_config->dpll_hw_state.cfgcr2); } else if (HAS_DDI(dev)) { - DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: wrpll: 0x%x\n", + DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: wrpll: 0x%x spll: 0x%x\n", pipe_config->ddi_pll_sel, - pipe_config->dpll_hw_state.wrpll); + pipe_config->dpll_hw_state.wrpll, + pipe_config->dpll_hw_state.spll); } else { DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, " "fp0: 0x%x, fp1: 0x%x\n", @@ -11593,56 +12192,6 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, } } -static bool encoders_cloneable(const struct intel_encoder *a, - const struct intel_encoder *b) -{ - /* masks could be asymmetric, so check both ways */ - return a == b || (a->cloneable & (1 << b->type) && - b->cloneable & (1 << a->type)); -} - -static bool check_single_encoder_cloning(struct drm_atomic_state *state, - struct intel_crtc *crtc, - struct intel_encoder *encoder) -{ - struct intel_encoder *source_encoder; - struct drm_connector *connector; - struct drm_connector_state *connector_state; - int i; - - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != &crtc->base) - continue; - - source_encoder = - to_intel_encoder(connector_state->best_encoder); - if (!encoders_cloneable(encoder, source_encoder)) - return false; - } - - return true; -} - -static bool check_encoder_cloning(struct drm_atomic_state *state, - struct intel_crtc *crtc) -{ - struct intel_encoder *encoder; - struct drm_connector *connector; - struct drm_connector_state *connector_state; - int i; - - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != &crtc->base) - continue; - - encoder = to_intel_encoder(connector_state->best_encoder); - if (!check_single_encoder_cloning(state, crtc, encoder)) - return false; - } - - return true; -} - static bool check_digital_port_conflicts(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; @@ -11696,6 +12245,7 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state) struct intel_dpll_hw_state dpll_hw_state; enum intel_dpll_id shared_dpll; uint32_t ddi_pll_sel; + bool force_thru; /* FIXME: before the switch to atomic started, a new pipe_config was * kzalloc'd. Code that depends on any field being zero should be @@ -11707,6 +12257,7 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state) shared_dpll = crtc_state->shared_dpll; dpll_hw_state = crtc_state->dpll_hw_state; ddi_pll_sel = crtc_state->ddi_pll_sel; + force_thru = crtc_state->pch_pfit.force_thru; memset(crtc_state, 0, sizeof *crtc_state); @@ -11715,13 +12266,14 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state) crtc_state->shared_dpll = shared_dpll; crtc_state->dpll_hw_state = dpll_hw_state; crtc_state->ddi_pll_sel = ddi_pll_sel; + crtc_state->pch_pfit.force_thru = force_thru; } static int intel_modeset_pipe_config(struct drm_crtc *crtc, - struct drm_atomic_state *state, struct intel_crtc_state *pipe_config) { + struct drm_atomic_state *state = pipe_config->base.state; struct intel_encoder *encoder; struct drm_connector *connector; struct drm_connector_state *connector_state; @@ -11729,16 +12281,6 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, int i; bool retry = true; - if (!check_encoder_cloning(state, to_intel_crtc(crtc))) { - DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); - return -EINVAL; - } - - if (!check_digital_port_conflicts(state)) { - DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n"); - return -EINVAL; - } - clear_intel_crtc_state(pipe_config); pipe_config->cpu_transcoder = @@ -11757,10 +12299,6 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC; - /* Compute a starting value for pipe_config->pipe_bpp taking the source - * plane pixel format and any sink constraints into account. Returns the - * source plane bpp so that dithering can be selected on mismatches - * after encoders and crtc also have had their say. */ base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc), pipe_config); if (base_bpp < 0) @@ -11829,93 +12367,42 @@ encoder_retry: /* Dithering seems to not pass-through bits correctly when it should, so * only enable it on 6bpc panels. */ pipe_config->dither = pipe_config->pipe_bpp == 6*3; - DRM_DEBUG_KMS("plane bpp: %i, pipe bpp: %i, dithering: %i\n", + DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n", base_bpp, pipe_config->pipe_bpp, pipe_config->dither); - return 0; fail: return ret; } -static bool intel_crtc_in_use(struct drm_crtc *crtc) -{ - struct drm_encoder *encoder; - struct drm_device *dev = crtc->dev; - - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) - if (encoder->crtc == crtc) - return true; - - return false; -} - -static bool -needs_modeset(struct drm_crtc_state *state) -{ - return state->mode_changed || state->active_changed; -} - static void -intel_modeset_update_state(struct drm_atomic_state *state) +intel_modeset_update_crtc_state(struct drm_atomic_state *state) { - struct drm_device *dev = state->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_encoder *intel_encoder; struct drm_crtc *crtc; struct drm_crtc_state *crtc_state; - struct drm_connector *connector; int i; - intel_shared_dpll_commit(dev_priv); - - for_each_intel_encoder(dev, intel_encoder) { - if (!intel_encoder->base.crtc) - continue; - - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (crtc != intel_encoder->base.crtc) - continue; - - if (crtc_state->enable && needs_modeset(crtc_state)) - intel_encoder->connectors_active = false; - - break; - } - } - - drm_atomic_helper_swap_state(state->dev, state); - intel_modeset_fixup_state(state); - /* Double check state. */ - for_each_crtc(dev, crtc) { - WARN_ON(crtc->state->enable != intel_crtc_in_use(crtc)); - } - - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - if (!connector->encoder || !connector->encoder->crtc) - continue; - - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (crtc != connector->encoder->crtc) - continue; - - if (crtc->state->enable && needs_modeset(crtc->state)) { - struct drm_property *dpms_property = - dev->mode_config.dpms_property; + for_each_crtc_in_state(state, crtc, crtc_state, i) { + to_intel_crtc(crtc)->config = to_intel_crtc_state(crtc->state); - connector->dpms = DRM_MODE_DPMS_ON; - drm_object_property_set_value(&connector->base, - dpms_property, - DRM_MODE_DPMS_ON); + /* Update hwmode for vblank functions */ + if (crtc->state->active) + crtc->hwmode = crtc->state->adjusted_mode; + else + crtc->hwmode.crtc_clock = 0; - intel_encoder = to_intel_encoder(connector->encoder); - intel_encoder->connectors_active = true; - } + /* + * Update legacy state to satisfy fbc code. This can + * be removed when fbc uses the atomic state. + */ + if (drm_atomic_get_existing_plane_state(state, crtc->primary)) { + struct drm_plane_state *plane_state = crtc->primary->state; - break; + crtc->primary->fb = plane_state->fb; + crtc->x = plane_state->src_x >> 16; + crtc->y = plane_state->src_y >> 16; } } - } static bool intel_fuzzy_clock_check(int clock1, int clock2) @@ -11940,29 +12427,134 @@ static bool intel_fuzzy_clock_check(int clock1, int clock2) list_for_each_entry((intel_crtc), \ &(dev)->mode_config.crtc_list, \ base.head) \ - if (mask & (1 <<(intel_crtc)->pipe)) + for_each_if (mask & (1 <<(intel_crtc)->pipe)) + +static bool +intel_compare_m_n(unsigned int m, unsigned int n, + unsigned int m2, unsigned int n2, + bool exact) +{ + if (m == m2 && n == n2) + return true; + + if (exact || !m || !n || !m2 || !n2) + return false; + + BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX); + + if (m > m2) { + while (m > m2) { + m2 <<= 1; + n2 <<= 1; + } + } else if (m < m2) { + while (m < m2) { + m <<= 1; + n <<= 1; + } + } + + return m == m2 && n == n2; +} + +static bool +intel_compare_link_m_n(const struct intel_link_m_n *m_n, + struct intel_link_m_n *m2_n2, + bool adjust) +{ + if (m_n->tu == m2_n2->tu && + intel_compare_m_n(m_n->gmch_m, m_n->gmch_n, + m2_n2->gmch_m, m2_n2->gmch_n, !adjust) && + intel_compare_m_n(m_n->link_m, m_n->link_n, + m2_n2->link_m, m2_n2->link_n, !adjust)) { + if (adjust) + *m2_n2 = *m_n; + + return true; + } + + return false; +} static bool intel_pipe_config_compare(struct drm_device *dev, struct intel_crtc_state *current_config, - struct intel_crtc_state *pipe_config) + struct intel_crtc_state *pipe_config, + bool adjust) { + bool ret = true; + +#define INTEL_ERR_OR_DBG_KMS(fmt, ...) \ + do { \ + if (!adjust) \ + DRM_ERROR(fmt, ##__VA_ARGS__); \ + else \ + DRM_DEBUG_KMS(fmt, ##__VA_ARGS__); \ + } while (0) + #define PIPE_CONF_CHECK_X(name) \ if (current_config->name != pipe_config->name) { \ - DRM_ERROR("mismatch in " #name " " \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \ "(expected 0x%08x, found 0x%08x)\n", \ current_config->name, \ pipe_config->name); \ - return false; \ + ret = false; \ } #define PIPE_CONF_CHECK_I(name) \ if (current_config->name != pipe_config->name) { \ - DRM_ERROR("mismatch in " #name " " \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \ "(expected %i, found %i)\n", \ current_config->name, \ pipe_config->name); \ - return false; \ + ret = false; \ + } + +#define PIPE_CONF_CHECK_M_N(name) \ + if (!intel_compare_link_m_n(¤t_config->name, \ + &pipe_config->name,\ + adjust)) { \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \ + "(expected tu %i gmch %i/%i link %i/%i, " \ + "found tu %i, gmch %i/%i link %i/%i)\n", \ + current_config->name.tu, \ + current_config->name.gmch_m, \ + current_config->name.gmch_n, \ + current_config->name.link_m, \ + current_config->name.link_n, \ + pipe_config->name.tu, \ + pipe_config->name.gmch_m, \ + pipe_config->name.gmch_n, \ + pipe_config->name.link_m, \ + pipe_config->name.link_n); \ + ret = false; \ + } + +#define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) \ + if (!intel_compare_link_m_n(¤t_config->name, \ + &pipe_config->name, adjust) && \ + !intel_compare_link_m_n(¤t_config->alt_name, \ + &pipe_config->name, adjust)) { \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \ + "(expected tu %i gmch %i/%i link %i/%i, " \ + "or tu %i gmch %i/%i link %i/%i, " \ + "found tu %i, gmch %i/%i link %i/%i)\n", \ + current_config->name.tu, \ + current_config->name.gmch_m, \ + current_config->name.gmch_n, \ + current_config->name.link_m, \ + current_config->name.link_n, \ + current_config->alt_name.tu, \ + current_config->alt_name.gmch_m, \ + current_config->alt_name.gmch_n, \ + current_config->alt_name.link_m, \ + current_config->alt_name.link_n, \ + pipe_config->name.tu, \ + pipe_config->name.gmch_m, \ + pipe_config->name.gmch_n, \ + pipe_config->name.link_m, \ + pipe_config->name.link_n); \ + ret = false; \ } /* This is required for BDW+ where there is only one set of registers for @@ -11973,30 +12565,30 @@ intel_pipe_config_compare(struct drm_device *dev, #define PIPE_CONF_CHECK_I_ALT(name, alt_name) \ if ((current_config->name != pipe_config->name) && \ (current_config->alt_name != pipe_config->name)) { \ - DRM_ERROR("mismatch in " #name " " \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \ "(expected %i or %i, found %i)\n", \ current_config->name, \ current_config->alt_name, \ pipe_config->name); \ - return false; \ + ret = false; \ } #define PIPE_CONF_CHECK_FLAGS(name, mask) \ if ((current_config->name ^ pipe_config->name) & (mask)) { \ - DRM_ERROR("mismatch in " #name "(" #mask ") " \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name "(" #mask ") " \ "(expected %i, found %i)\n", \ current_config->name & (mask), \ pipe_config->name & (mask)); \ - return false; \ + ret = false; \ } #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) \ if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \ - DRM_ERROR("mismatch in " #name " " \ + INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \ "(expected %i, found %i)\n", \ current_config->name, \ pipe_config->name); \ - return false; \ + ret = false; \ } #define PIPE_CONF_QUIRK(quirk) \ @@ -12006,35 +12598,20 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_I(has_pch_encoder); PIPE_CONF_CHECK_I(fdi_lanes); - PIPE_CONF_CHECK_I(fdi_m_n.gmch_m); - PIPE_CONF_CHECK_I(fdi_m_n.gmch_n); - PIPE_CONF_CHECK_I(fdi_m_n.link_m); - PIPE_CONF_CHECK_I(fdi_m_n.link_n); - PIPE_CONF_CHECK_I(fdi_m_n.tu); + PIPE_CONF_CHECK_M_N(fdi_m_n); PIPE_CONF_CHECK_I(has_dp_encoder); + PIPE_CONF_CHECK_I(lane_count); if (INTEL_INFO(dev)->gen < 8) { - PIPE_CONF_CHECK_I(dp_m_n.gmch_m); - PIPE_CONF_CHECK_I(dp_m_n.gmch_n); - PIPE_CONF_CHECK_I(dp_m_n.link_m); - PIPE_CONF_CHECK_I(dp_m_n.link_n); - PIPE_CONF_CHECK_I(dp_m_n.tu); - - if (current_config->has_drrs) { - PIPE_CONF_CHECK_I(dp_m2_n2.gmch_m); - PIPE_CONF_CHECK_I(dp_m2_n2.gmch_n); - PIPE_CONF_CHECK_I(dp_m2_n2.link_m); - PIPE_CONF_CHECK_I(dp_m2_n2.link_n); - PIPE_CONF_CHECK_I(dp_m2_n2.tu); - } - } else { - PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_m, dp_m2_n2.gmch_m); - PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_n, dp_m2_n2.gmch_n); - PIPE_CONF_CHECK_I_ALT(dp_m_n.link_m, dp_m2_n2.link_m); - PIPE_CONF_CHECK_I_ALT(dp_m_n.link_n, dp_m2_n2.link_n); - PIPE_CONF_CHECK_I_ALT(dp_m_n.tu, dp_m2_n2.tu); - } + PIPE_CONF_CHECK_M_N(dp_m_n); + + if (current_config->has_drrs) + PIPE_CONF_CHECK_M_N(dp_m2_n2); + } else + PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2); + + PIPE_CONF_CHECK_I(has_dsi_encoder); PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay); PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal); @@ -12073,32 +12650,24 @@ intel_pipe_config_compare(struct drm_device *dev, DRM_MODE_FLAG_NVSYNC); } - PIPE_CONF_CHECK_I(pipe_src_w); - PIPE_CONF_CHECK_I(pipe_src_h); + PIPE_CONF_CHECK_X(gmch_pfit.control); + /* pfit ratios are autocomputed by the hw on gen4+ */ + if (INTEL_INFO(dev)->gen < 4) + PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios); + PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits); - /* - * FIXME: BIOS likes to set up a cloned config with lvds+external - * screen. Since we don't yet re-compute the pipe config when moving - * just the lvds port away to another pipe the sw tracking won't match. - * - * Proper atomic modesets with recomputed global state will fix this. - * Until then just don't check gmch state for inherited modes. - */ - if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_INHERITED_MODE)) { - PIPE_CONF_CHECK_I(gmch_pfit.control); - /* pfit ratios are autocomputed by the hw on gen4+ */ - if (INTEL_INFO(dev)->gen < 4) - PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios); - PIPE_CONF_CHECK_I(gmch_pfit.lvds_border_bits); - } + if (!adjust) { + PIPE_CONF_CHECK_I(pipe_src_w); + PIPE_CONF_CHECK_I(pipe_src_h); - PIPE_CONF_CHECK_I(pch_pfit.enabled); - if (current_config->pch_pfit.enabled) { - PIPE_CONF_CHECK_I(pch_pfit.pos); - PIPE_CONF_CHECK_I(pch_pfit.size); - } + PIPE_CONF_CHECK_I(pch_pfit.enabled); + if (current_config->pch_pfit.enabled) { + PIPE_CONF_CHECK_X(pch_pfit.pos); + PIPE_CONF_CHECK_X(pch_pfit.size); + } - PIPE_CONF_CHECK_I(scaler_state.scaler_id); + PIPE_CONF_CHECK_I(scaler_state.scaler_id); + } /* BDW+ don't expose a synchronous way to read the state */ if (IS_HASWELL(dev)) @@ -12114,6 +12683,7 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_X(dpll_hw_state.fp0); PIPE_CONF_CHECK_X(dpll_hw_state.fp1); PIPE_CONF_CHECK_X(dpll_hw_state.wrpll); + PIPE_CONF_CHECK_X(dpll_hw_state.spll); PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1); PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1); PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2); @@ -12130,8 +12700,9 @@ intel_pipe_config_compare(struct drm_device *dev, #undef PIPE_CONF_CHECK_FLAGS #undef PIPE_CONF_CHECK_CLOCK_FUZZY #undef PIPE_CONF_QUIRK +#undef INTEL_ERR_OR_DBG_KMS - return true; + return ret; } static void check_wm_state(struct drm_device *dev) @@ -12170,8 +12741,8 @@ static void check_wm_state(struct drm_device *dev) } /* cursor */ - hw_entry = &hw_ddb.cursor[pipe]; - sw_entry = &sw_ddb->cursor[pipe]; + hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR]; + sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR]; if (skl_ddb_entry_equal(hw_entry, sw_entry)) continue; @@ -12185,17 +12756,23 @@ static void check_wm_state(struct drm_device *dev) } static void -check_connector_state(struct drm_device *dev) +check_connector_state(struct drm_device *dev, + struct drm_atomic_state *old_state) { - struct intel_connector *connector; + struct drm_connector_state *old_conn_state; + struct drm_connector *connector; + int i; + + for_each_connector_in_state(old_state, connector, old_conn_state, i) { + struct drm_encoder *encoder = connector->encoder; + struct drm_connector_state *state = connector->state; - for_each_intel_connector(dev, connector) { /* This also checks the encoder/connector hw state with the * ->get_hw_state callbacks. */ - intel_connector_check_state(connector); + intel_connector_check_state(to_intel_connector(connector)); - I915_STATE_WARN(&connector->new_encoder->base != connector->base.encoder, - "connector's staged encoder doesn't match current encoder\n"); + I915_STATE_WARN(state->best_encoder != encoder, + "connector's atomic encoder doesn't match legacy encoder\n"); } } @@ -12207,124 +12784,107 @@ check_encoder_state(struct drm_device *dev) for_each_intel_encoder(dev, encoder) { bool enabled = false; - bool active = false; - enum pipe pipe, tracked_pipe; + enum pipe pipe; DRM_DEBUG_KMS("[ENCODER:%d:%s]\n", encoder->base.base.id, encoder->base.name); - I915_STATE_WARN(&encoder->new_crtc->base != encoder->base.crtc, - "encoder's stage crtc doesn't match current crtc\n"); - I915_STATE_WARN(encoder->connectors_active && !encoder->base.crtc, - "encoder's active_connectors set, but no crtc\n"); - for_each_intel_connector(dev, connector) { - if (connector->base.encoder != &encoder->base) + if (connector->base.state->best_encoder != &encoder->base) continue; enabled = true; - if (connector->base.dpms != DRM_MODE_DPMS_OFF) - active = true; + + I915_STATE_WARN(connector->base.state->crtc != + encoder->base.crtc, + "connector's crtc doesn't match encoder crtc\n"); } - /* - * for MST connectors if we unplug the connector is gone - * away but the encoder is still connected to a crtc - * until a modeset happens in response to the hotplug. - */ - if (!enabled && encoder->base.encoder_type == DRM_MODE_ENCODER_DPMST) - continue; I915_STATE_WARN(!!encoder->base.crtc != enabled, "encoder's enabled state mismatch " "(expected %i, found %i)\n", !!encoder->base.crtc, enabled); - I915_STATE_WARN(active && !encoder->base.crtc, - "active encoder with no crtc\n"); - I915_STATE_WARN(encoder->connectors_active != active, - "encoder's computed active state doesn't match tracked active state " - "(expected %i, found %i)\n", active, encoder->connectors_active); - - active = encoder->get_hw_state(encoder, &pipe); - I915_STATE_WARN(active != encoder->connectors_active, - "encoder's hw state doesn't match sw tracking " - "(expected %i, found %i)\n", - encoder->connectors_active, active); - - if (!encoder->base.crtc) - continue; - - tracked_pipe = to_intel_crtc(encoder->base.crtc)->pipe; - I915_STATE_WARN(active && pipe != tracked_pipe, - "active encoder's pipe doesn't match" - "(expected %i, found %i)\n", - tracked_pipe, pipe); + if (!encoder->base.crtc) { + bool active; + active = encoder->get_hw_state(encoder, &pipe); + I915_STATE_WARN(active, + "encoder detached but still enabled on pipe %c.\n", + pipe_name(pipe)); + } } } static void -check_crtc_state(struct drm_device *dev) +check_crtc_state(struct drm_device *dev, struct drm_atomic_state *old_state) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *crtc; struct intel_encoder *encoder; - struct intel_crtc_state pipe_config; + struct drm_crtc_state *old_crtc_state; + struct drm_crtc *crtc; + int i; - for_each_intel_crtc(dev, crtc) { - bool enabled = false; - bool active = false; + for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) { + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc_state *pipe_config, *sw_config; + bool active; - memset(&pipe_config, 0, sizeof(pipe_config)); + if (!needs_modeset(crtc->state) && + !to_intel_crtc_state(crtc->state)->update_pipe) + continue; - DRM_DEBUG_KMS("[CRTC:%d]\n", - crtc->base.base.id); + __drm_atomic_helper_crtc_destroy_state(crtc, old_crtc_state); + pipe_config = to_intel_crtc_state(old_crtc_state); + memset(pipe_config, 0, sizeof(*pipe_config)); + pipe_config->base.crtc = crtc; + pipe_config->base.state = old_state; - I915_STATE_WARN(crtc->active && !crtc->base.state->enable, - "active crtc, but not enabled in sw tracking\n"); + DRM_DEBUG_KMS("[CRTC:%d]\n", + crtc->base.id); - for_each_intel_encoder(dev, encoder) { - if (encoder->base.crtc != &crtc->base) - continue; - enabled = true; - if (encoder->connectors_active) - active = true; - } + active = dev_priv->display.get_pipe_config(intel_crtc, + pipe_config); - I915_STATE_WARN(active != crtc->active, - "crtc's computed active state doesn't match tracked active state " - "(expected %i, found %i)\n", active, crtc->active); - I915_STATE_WARN(enabled != crtc->base.state->enable, - "crtc's computed enabled state doesn't match tracked enabled state " - "(expected %i, found %i)\n", enabled, - crtc->base.state->enable); + /* hw state is inconsistent with the pipe quirk */ + if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || + (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) + active = crtc->state->active; - active = dev_priv->display.get_pipe_config(crtc, - &pipe_config); + I915_STATE_WARN(crtc->state->active != active, + "crtc active state doesn't match with hw state " + "(expected %i, found %i)\n", crtc->state->active, active); - /* hw state is inconsistent with the pipe quirk */ - if ((crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) || - (crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)) - active = crtc->active; + I915_STATE_WARN(intel_crtc->active != crtc->state->active, + "transitional active state does not match atomic hw state " + "(expected %i, found %i)\n", crtc->state->active, intel_crtc->active); - for_each_intel_encoder(dev, encoder) { + for_each_encoder_on_crtc(dev, crtc, encoder) { enum pipe pipe; - if (encoder->base.crtc != &crtc->base) - continue; - if (encoder->get_hw_state(encoder, &pipe)) - encoder->get_config(encoder, &pipe_config); + + active = encoder->get_hw_state(encoder, &pipe); + I915_STATE_WARN(active != crtc->state->active, + "[ENCODER:%i] active %i with crtc active %i\n", + encoder->base.base.id, active, crtc->state->active); + + I915_STATE_WARN(active && intel_crtc->pipe != pipe, + "Encoder connected to wrong pipe %c\n", + pipe_name(pipe)); + + if (active) + encoder->get_config(encoder, pipe_config); } - I915_STATE_WARN(crtc->active != active, - "crtc active state doesn't match with hw state " - "(expected %i, found %i)\n", crtc->active, active); + if (!crtc->state->active) + continue; - if (active && - !intel_pipe_config_compare(dev, crtc->config, &pipe_config)) { + sw_config = to_intel_crtc_state(crtc->state); + if (!intel_pipe_config_compare(dev, sw_config, + pipe_config, false)) { I915_STATE_WARN(1, "pipe state doesn't match!\n"); - intel_dump_pipe_config(crtc, &pipe_config, + intel_dump_pipe_config(intel_crtc, pipe_config, "[hw state]"); - intel_dump_pipe_config(crtc, crtc->config, + intel_dump_pipe_config(intel_crtc, sw_config, "[sw state]"); } } @@ -12379,13 +12939,14 @@ check_shared_dpll_state(struct drm_device *dev) } } -void -intel_modeset_check_state(struct drm_device *dev) +static void +intel_modeset_check_state(struct drm_device *dev, + struct drm_atomic_state *old_state) { check_wm_state(dev); - check_connector_state(dev); + check_connector_state(dev, old_state); check_encoder_state(dev); - check_crtc_state(dev); + check_crtc_state(dev, old_state); check_shared_dpll_state(dev); } @@ -12424,11 +12985,11 @@ static void update_scanline_offset(struct intel_crtc *crtc) * one to the value. */ if (IS_GEN2(dev)) { - const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; int vtotal; - vtotal = mode->crtc_vtotal; - if (mode->flags & DRM_MODE_FLAG_INTERLACE) + vtotal = adjusted_mode->crtc_vtotal; + if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) vtotal /= 2; crtc->scanline_offset = vtotal - 1; @@ -12439,113 +13000,144 @@ static void update_scanline_offset(struct intel_crtc *crtc) crtc->scanline_offset = 1; } -static struct intel_crtc_state * -intel_modeset_compute_config(struct drm_crtc *crtc, - struct drm_atomic_state *state) +static void intel_modeset_clear_plls(struct drm_atomic_state *state) { - struct intel_crtc_state *pipe_config; - int ret = 0; - - ret = drm_atomic_add_affected_connectors(state, crtc); - if (ret) - return ERR_PTR(ret); - - ret = drm_atomic_helper_check_modeset(state->dev, state); - if (ret) - return ERR_PTR(ret); - - /* - * Note this needs changes when we start tracking multiple modes - * and crtcs. At that point we'll need to compute the whole config - * (i.e. one pipe_config for each crtc) rather than just the one - * for this crtc. - */ - pipe_config = intel_atomic_get_crtc_state(state, to_intel_crtc(crtc)); - if (IS_ERR(pipe_config)) - return pipe_config; + struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_shared_dpll_config *shared_dpll = NULL; + struct intel_crtc *intel_crtc; + struct intel_crtc_state *intel_crtc_state; + struct drm_crtc *crtc; + struct drm_crtc_state *crtc_state; + int i; - if (!pipe_config->base.enable) - return pipe_config; + if (!dev_priv->display.crtc_compute_clock) + return; - ret = intel_modeset_pipe_config(crtc, state, pipe_config); - if (ret) - return ERR_PTR(ret); + for_each_crtc_in_state(state, crtc, crtc_state, i) { + int dpll; - /* Check things that can only be changed through modeset */ - if (pipe_config->has_audio != - to_intel_crtc(crtc)->config->has_audio) - pipe_config->base.mode_changed = true; + intel_crtc = to_intel_crtc(crtc); + intel_crtc_state = to_intel_crtc_state(crtc_state); + dpll = intel_crtc_state->shared_dpll; - /* - * Note we have an issue here with infoframes: current code - * only updates them on the full mode set path per hw - * requirements. So here we should be checking for any - * required changes and forcing a mode set. - */ + if (!needs_modeset(crtc_state) || dpll == DPLL_ID_PRIVATE) + continue; - intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,"[modeset]"); + intel_crtc_state->shared_dpll = DPLL_ID_PRIVATE; - ret = drm_atomic_helper_check_planes(state->dev, state); - if (ret) - return ERR_PTR(ret); + if (!shared_dpll) + shared_dpll = intel_atomic_get_shared_dpll_state(state); - return pipe_config; + shared_dpll[dpll].crtc_mask &= ~(1 << intel_crtc->pipe); + } } -static int __intel_set_mode_setup_plls(struct drm_atomic_state *state) +/* + * This implements the workaround described in the "notes" section of the mode + * set sequence documentation. When going from no pipes or single pipe to + * multiple pipes, and planes are enabled after the pipe, we need to wait at + * least 2 vblanks on the first pipe before enabling planes on the second pipe. + */ +static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state) { - struct drm_device *dev = state->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - unsigned clear_pipes = 0; + struct drm_crtc_state *crtc_state; struct intel_crtc *intel_crtc; - struct intel_crtc_state *intel_crtc_state; struct drm_crtc *crtc; - struct drm_crtc_state *crtc_state; - int ret = 0; + struct intel_crtc_state *first_crtc_state = NULL; + struct intel_crtc_state *other_crtc_state = NULL; + enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE; int i; - if (!dev_priv->display.crtc_compute_clock) - return 0; - + /* look at all crtc's that are going to be enabled in during modeset */ for_each_crtc_in_state(state, crtc, crtc_state, i) { intel_crtc = to_intel_crtc(crtc); - intel_crtc_state = to_intel_crtc_state(crtc_state); - if (needs_modeset(crtc_state)) { - clear_pipes |= 1 << intel_crtc->pipe; - intel_crtc_state->shared_dpll = DPLL_ID_PRIVATE; + if (!crtc_state->active || !needs_modeset(crtc_state)) + continue; + + if (first_crtc_state) { + other_crtc_state = to_intel_crtc_state(crtc_state); + break; + } else { + first_crtc_state = to_intel_crtc_state(crtc_state); + first_pipe = intel_crtc->pipe; } } - ret = intel_shared_dpll_start_config(dev_priv, clear_pipes); - if (ret) - goto done; + /* No workaround needed? */ + if (!first_crtc_state) + return 0; - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (!needs_modeset(crtc_state) || !crtc_state->enable) + /* w/a possibly needed, check how many crtc's are already enabled. */ + for_each_intel_crtc(state->dev, intel_crtc) { + struct intel_crtc_state *pipe_config; + + pipe_config = intel_atomic_get_crtc_state(state, intel_crtc); + if (IS_ERR(pipe_config)) + return PTR_ERR(pipe_config); + + pipe_config->hsw_workaround_pipe = INVALID_PIPE; + + if (!pipe_config->base.active || + needs_modeset(&pipe_config->base)) continue; - intel_crtc = to_intel_crtc(crtc); - intel_crtc_state = to_intel_crtc_state(crtc_state); + /* 2 or more enabled crtcs means no need for w/a */ + if (enabled_pipe != INVALID_PIPE) + return 0; - ret = dev_priv->display.crtc_compute_clock(intel_crtc, - intel_crtc_state); - if (ret) { - intel_shared_dpll_abort_config(dev_priv); - goto done; - } + enabled_pipe = intel_crtc->pipe; + } + + if (enabled_pipe != INVALID_PIPE) + first_crtc_state->hsw_workaround_pipe = enabled_pipe; + else if (other_crtc_state) + other_crtc_state->hsw_workaround_pipe = first_pipe; + + return 0; +} + +static int intel_modeset_all_pipes(struct drm_atomic_state *state) +{ + struct drm_crtc *crtc; + struct drm_crtc_state *crtc_state; + int ret = 0; + + /* add all active pipes to the state */ + for_each_crtc(state->dev, crtc) { + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + if (!crtc_state->active || needs_modeset(crtc_state)) + continue; + + crtc_state->mode_changed = true; + + ret = drm_atomic_add_affected_connectors(state, crtc); + if (ret) + break; + + ret = drm_atomic_add_affected_planes(state, crtc); + if (ret) + break; } -done: return ret; } -/* Code that should eventually be part of atomic_check() */ -static int __intel_set_mode_checks(struct drm_atomic_state *state) +static int intel_modeset_checks(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = dev->dev_private; int ret; + if (!check_digital_port_conflicts(state)) { + DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n"); + return -EINVAL; + } + /* * See if the config requires any additional preparation, e.g. * to adjust global state with pipes off. We need to do this @@ -12553,405 +13145,389 @@ static int __intel_set_mode_checks(struct drm_atomic_state *state) * mode set on this crtc. For other crtcs we need to use the * adjusted_mode bits in the crtc directly. */ - if (IS_VALLEYVIEW(dev) || IS_BROXTON(dev)) { - ret = valleyview_modeset_global_pipes(state); - if (ret) + if (dev_priv->display.modeset_calc_cdclk) { + unsigned int cdclk; + + ret = dev_priv->display.modeset_calc_cdclk(state); + + cdclk = to_intel_atomic_state(state)->cdclk; + if (!ret && cdclk != dev_priv->cdclk_freq) + ret = intel_modeset_all_pipes(state); + + if (ret < 0) return ret; - } + } else + to_intel_atomic_state(state)->cdclk = dev_priv->cdclk_freq; - ret = __intel_set_mode_setup_plls(state); - if (ret) - return ret; + intel_modeset_clear_plls(state); + + if (IS_HASWELL(dev)) + return haswell_mode_set_planes_workaround(state); return 0; } -static int __intel_set_mode(struct drm_crtc *modeset_crtc, - struct intel_crtc_state *pipe_config) +/* + * Handle calculation of various watermark data at the end of the atomic check + * phase. The code here should be run after the per-crtc and per-plane 'check' + * handlers to ensure that all derived state has been updated. + */ +static void calc_watermark_data(struct drm_atomic_state *state) { - struct drm_device *dev = modeset_crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_atomic_state *state = pipe_config->base.state; + struct drm_device *dev = state->dev; + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); struct drm_crtc *crtc; - struct drm_crtc_state *crtc_state; - int ret = 0; - int i; + struct drm_crtc_state *cstate; + struct drm_plane *plane; + struct drm_plane_state *pstate; - ret = __intel_set_mode_checks(state); - if (ret < 0) - return ret; + /* + * Calculate watermark configuration details now that derived + * plane/crtc state is all properly updated. + */ + drm_for_each_crtc(crtc, dev) { + cstate = drm_atomic_get_existing_crtc_state(state, crtc) ?: + crtc->state; - ret = drm_atomic_helper_prepare_planes(dev, state); - if (ret) - return ret; + if (cstate->active) + intel_state->wm_config.num_pipes_active++; + } + drm_for_each_legacy_plane(plane, dev) { + pstate = drm_atomic_get_existing_plane_state(state, plane) ?: + plane->state; - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (!needs_modeset(crtc_state)) + if (!to_intel_plane_state(pstate)->visible) continue; - if (!crtc_state->enable) { - if (crtc->state->enable) - intel_crtc_disable(crtc); - } else if (crtc->state->enable) { - intel_crtc_disable_planes(crtc); - dev_priv->display.crtc_disable(crtc); - } + intel_state->wm_config.sprites_enabled = true; + if (pstate->crtc_w != pstate->src_w >> 16 || + pstate->crtc_h != pstate->src_h >> 16) + intel_state->wm_config.sprites_scaled = true; } +} - /* crtc->mode is already used by the ->mode_set callbacks, hence we need - * to set it here already despite that we pass it down the callchain. - * - * Note we'll need to fix this up when we start tracking multiple - * pipes; here we assume a single modeset_pipe and only track the - * single crtc and mode. - */ - if (pipe_config->base.enable && needs_modeset(&pipe_config->base)) { - modeset_crtc->mode = pipe_config->base.mode; +/** + * intel_atomic_check - validate state object + * @dev: drm device + * @state: state to validate + */ +static int intel_atomic_check(struct drm_device *dev, + struct drm_atomic_state *state) +{ + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); + struct drm_crtc *crtc; + struct drm_crtc_state *crtc_state; + int ret, i; + bool any_ms = false; - /* - * Calculate and store various constants which - * are later needed by vblank and swap-completion - * timestamping. They are derived from true hwmode. - */ - drm_calc_timestamping_constants(modeset_crtc, - &pipe_config->base.adjusted_mode); - } + ret = drm_atomic_helper_check_modeset(dev, state); + if (ret) + return ret; - /* Only after disabling all output pipelines that will be changed can we - * update the the output configuration. */ - intel_modeset_update_state(state); + for_each_crtc_in_state(state, crtc, crtc_state, i) { + struct intel_crtc_state *pipe_config = + to_intel_crtc_state(crtc_state); - /* The state has been swaped above, so state actually contains the - * old state now. */ + memset(&to_intel_crtc(crtc)->atomic, 0, + sizeof(struct intel_crtc_atomic_commit)); - modeset_update_crtc_power_domains(state); + /* Catch I915_MODE_FLAG_INHERITED */ + if (crtc_state->mode.private_flags != crtc->state->mode.private_flags) + crtc_state->mode_changed = true; - /* Now enable the clocks, plane, pipe, and connectors that we set up. */ - for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (!needs_modeset(crtc->state) || !crtc->state->enable) { - drm_atomic_helper_commit_planes_on_crtc(crtc_state); + if (!crtc_state->enable) { + if (needs_modeset(crtc_state)) + any_ms = true; continue; } - update_scanline_offset(to_intel_crtc(crtc)); - - dev_priv->display.crtc_enable(crtc); - drm_atomic_helper_commit_planes_on_crtc(crtc_state); - } + if (!needs_modeset(crtc_state)) + continue; - /* FIXME: add subpixel order */ + /* FIXME: For only active_changed we shouldn't need to do any + * state recomputation at all. */ - drm_atomic_helper_cleanup_planes(dev, state); + ret = drm_atomic_add_affected_connectors(state, crtc); + if (ret) + return ret; - drm_atomic_state_free(state); + ret = intel_modeset_pipe_config(crtc, pipe_config); + if (ret) + return ret; - return 0; -} + if (i915.fastboot && + intel_pipe_config_compare(state->dev, + to_intel_crtc_state(crtc->state), + pipe_config, true)) { + crtc_state->mode_changed = false; + to_intel_crtc_state(crtc_state)->update_pipe = true; + } -static int intel_set_mode_with_config(struct drm_crtc *crtc, - struct intel_crtc_state *pipe_config, - bool force_restore) -{ - int ret; + if (needs_modeset(crtc_state)) { + any_ms = true; - ret = __intel_set_mode(crtc, pipe_config); + ret = drm_atomic_add_affected_planes(state, crtc); + if (ret) + return ret; + } - if (ret == 0 && force_restore) { - intel_modeset_update_staged_output_state(crtc->dev); - intel_modeset_check_state(crtc->dev); + intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config, + needs_modeset(crtc_state) ? + "[modeset]" : "[fastset]"); } - return ret; -} - -static int intel_set_mode(struct drm_crtc *crtc, - struct drm_atomic_state *state, - bool force_restore) -{ - struct intel_crtc_state *pipe_config; - int ret = 0; + if (any_ms) { + ret = intel_modeset_checks(state); - pipe_config = intel_modeset_compute_config(crtc, state); - if (IS_ERR(pipe_config)) { - ret = PTR_ERR(pipe_config); - goto out; - } + if (ret) + return ret; + } else + intel_state->cdclk = to_i915(state->dev)->cdclk_freq; - ret = intel_set_mode_with_config(crtc, pipe_config, force_restore); + ret = drm_atomic_helper_check_planes(state->dev, state); if (ret) - goto out; + return ret; -out: - return ret; + calc_watermark_data(state); + + return 0; } -void intel_crtc_restore_mode(struct drm_crtc *crtc) +static int intel_atomic_prepare_commit(struct drm_device *dev, + struct drm_atomic_state *state, + bool async) { - struct drm_device *dev = crtc->dev; - struct drm_atomic_state *state; - struct intel_encoder *encoder; - struct intel_connector *connector; - struct drm_connector_state *connector_state; - struct intel_crtc_state *crtc_state; - int ret; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, ret; - state = drm_atomic_state_alloc(dev); - if (!state) { - DRM_DEBUG_KMS("[CRTC:%d] mode restore failed, out of memory", - crtc->base.id); - return; + if (async) { + DRM_DEBUG_KMS("i915 does not yet support async commit\n"); + return -EINVAL; } - state->acquire_ctx = dev->mode_config.acquire_ctx; - - /* The force restore path in the HW readout code relies on the staged - * config still keeping the user requested config while the actual - * state has been overwritten by the configuration read from HW. We - * need to copy the staged config to the atomic state, otherwise the - * mode set will just reapply the state the HW is already in. */ - for_each_intel_encoder(dev, encoder) { - if (&encoder->new_crtc->base != crtc) - continue; + for_each_crtc_in_state(state, crtc, crtc_state, i) { + ret = intel_crtc_wait_for_pending_flips(crtc); + if (ret) + return ret; - for_each_intel_connector(dev, connector) { - if (connector->new_encoder != encoder) - continue; + if (atomic_read(&to_intel_crtc(crtc)->unpin_work_count) >= 2) + flush_workqueue(dev_priv->wq); + } - connector_state = drm_atomic_get_connector_state(state, &connector->base); - if (IS_ERR(connector_state)) { - DRM_DEBUG_KMS("Failed to add [CONNECTOR:%d:%s] to state: %ld\n", - connector->base.base.id, - connector->base.name, - PTR_ERR(connector_state)); - continue; - } + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; - connector_state->crtc = crtc; - connector_state->best_encoder = &encoder->base; - } - } + ret = drm_atomic_helper_prepare_planes(dev, state); + if (!ret && !async && !i915_reset_in_progress(&dev_priv->gpu_error)) { + u32 reset_counter; - crtc_state = intel_atomic_get_crtc_state(state, to_intel_crtc(crtc)); - if (IS_ERR(crtc_state)) { - DRM_DEBUG_KMS("Failed to add [CRTC:%d] to state: %ld\n", - crtc->base.id, PTR_ERR(crtc_state)); - drm_atomic_state_free(state); - return; - } + reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); + mutex_unlock(&dev->struct_mutex); - crtc_state->base.active = crtc_state->base.enable = - to_intel_crtc(crtc)->new_enabled; + for_each_plane_in_state(state, plane, plane_state, i) { + struct intel_plane_state *intel_plane_state = + to_intel_plane_state(plane_state); - drm_mode_copy(&crtc_state->base.mode, &crtc->mode); + if (!intel_plane_state->wait_req) + continue; - intel_modeset_setup_plane_state(state, crtc, &crtc->mode, - crtc->primary->fb, crtc->x, crtc->y); + ret = __i915_wait_request(intel_plane_state->wait_req, + reset_counter, true, + NULL, NULL); - ret = intel_set_mode(crtc, state, false); - if (ret) - drm_atomic_state_free(state); -} + /* Swallow -EIO errors to allow updates during hw lockup. */ + if (ret == -EIO) + ret = 0; -#undef for_each_intel_crtc_masked + if (ret) + break; + } -static bool intel_connector_in_mode_set(struct intel_connector *connector, - struct drm_mode_set *set) -{ - int ro; + if (!ret) + return 0; - for (ro = 0; ro < set->num_connectors; ro++) - if (set->connectors[ro] == &connector->base) - return true; + mutex_lock(&dev->struct_mutex); + drm_atomic_helper_cleanup_planes(dev, state); + } - return false; + mutex_unlock(&dev->struct_mutex); + return ret; } -static int -intel_modeset_stage_output_state(struct drm_device *dev, - struct drm_mode_set *set, - struct drm_atomic_state *state) +/** + * intel_atomic_commit - commit validated state object + * @dev: DRM device + * @state: the top-level driver state object + * @async: asynchronous commit + * + * This function commits a top-level state object that has been validated + * with drm_atomic_helper_check(). + * + * FIXME: Atomic modeset support for i915 is not yet complete. At the moment + * we can only handle plane-related operations and do not yet support + * asynchronous commit. + * + * RETURNS + * Zero for success or -errno. + */ +static int intel_atomic_commit(struct drm_device *dev, + struct drm_atomic_state *state, + bool async) { - struct intel_connector *connector; - struct drm_connector *drm_connector; - struct drm_connector_state *connector_state; - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc_state *crtc_state; - int i, ret; - - /* The upper layers ensure that we either disable a crtc or have a list - * of connectors. For paranoia, double-check this. */ - WARN_ON(!set->fb && (set->num_connectors != 0)); - WARN_ON(set->fb && (set->num_connectors == 0)); - - for_each_intel_connector(dev, connector) { - bool in_mode_set = intel_connector_in_mode_set(connector, set); + struct drm_crtc *crtc; + int ret = 0; + int i; + bool any_ms = false; - if (!in_mode_set && connector->base.state->crtc != set->crtc) - continue; + ret = intel_atomic_prepare_commit(dev, state, async); + if (ret) { + DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); + return ret; + } - connector_state = - drm_atomic_get_connector_state(state, &connector->base); - if (IS_ERR(connector_state)) - return PTR_ERR(connector_state); + drm_atomic_helper_swap_state(dev, state); + dev_priv->wm.config = to_intel_atomic_state(state)->wm_config; - if (in_mode_set) { - int pipe = to_intel_crtc(set->crtc)->pipe; - connector_state->best_encoder = - &intel_find_encoder(connector, pipe)->base; - } + for_each_crtc_in_state(state, crtc, crtc_state, i) { + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - if (connector->base.state->crtc != set->crtc) + if (!needs_modeset(crtc->state)) continue; - /* If we disable the crtc, disable all its connectors. Also, if - * the connector is on the changing crtc but not on the new - * connector list, disable it. */ - if (!set->fb || !in_mode_set) { - connector_state->best_encoder = NULL; + any_ms = true; + intel_pre_plane_update(intel_crtc); + + if (crtc_state->active) { + intel_crtc_disable_planes(crtc, crtc_state->plane_mask); + dev_priv->display.crtc_disable(crtc); + intel_crtc->active = false; + intel_disable_shared_dpll(intel_crtc); - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [NOCRTC]\n", - connector->base.base.id, - connector->base.name); + /* + * Underruns don't always raise + * interrupts, so check manually. + */ + intel_check_cpu_fifo_underruns(dev_priv); + intel_check_pch_fifo_underruns(dev_priv); } } - /* connector->new_encoder is now updated for all connectors. */ - for_each_connector_in_state(state, drm_connector, connector_state, i) { - connector = to_intel_connector(drm_connector); + /* Only after disabling all output pipelines that will be changed can we + * update the the output configuration. */ + intel_modeset_update_crtc_state(state); - if (!connector_state->best_encoder) { - ret = drm_atomic_set_crtc_for_connector(connector_state, - NULL); - if (ret) - return ret; + if (any_ms) { + intel_shared_dpll_commit(state); - continue; - } + drm_atomic_helper_update_legacy_modeset_state(state->dev, state); + modeset_update_crtc_power_domains(state); + } - if (intel_connector_in_mode_set(connector, set)) { - struct drm_crtc *crtc = connector->base.state->crtc; + /* Now enable the clocks, plane, pipe, and connectors that we set up. */ + for_each_crtc_in_state(state, crtc, crtc_state, i) { + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + bool modeset = needs_modeset(crtc->state); + bool update_pipe = !modeset && + to_intel_crtc_state(crtc->state)->update_pipe; + unsigned long put_domains = 0; - /* If this connector was in a previous crtc, add it - * to the state. We might need to disable it. */ - if (crtc) { - crtc_state = - drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); - } + if (modeset) + intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET); - ret = drm_atomic_set_crtc_for_connector(connector_state, - set->crtc); - if (ret) - return ret; + if (modeset && crtc->state->active) { + update_scanline_offset(to_intel_crtc(crtc)); + dev_priv->display.crtc_enable(crtc); } - /* Make sure the new CRTC will work with the encoder */ - if (!drm_encoder_crtc_ok(connector_state->best_encoder, - connector_state->crtc)) { - return -EINVAL; + if (update_pipe) { + put_domains = modeset_get_crtc_power_domains(crtc); + + /* make sure intel_modeset_check_state runs */ + any_ms = true; } - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d]\n", - connector->base.base.id, - connector->base.name, - connector_state->crtc->base.id); + if (!modeset) + intel_pre_plane_update(intel_crtc); - if (connector_state->best_encoder != &connector->encoder->base) - connector->encoder = - to_intel_encoder(connector_state->best_encoder); - } + if (crtc->state->active && + (crtc->state->planes_changed || update_pipe)) + drm_atomic_helper_commit_planes_on_crtc(crtc_state); - for_each_crtc_in_state(state, crtc, crtc_state, i) { - bool has_connectors; + if (put_domains) + modeset_put_power_domains(dev_priv, put_domains); - ret = drm_atomic_add_affected_connectors(state, crtc); - if (ret) - return ret; + intel_post_plane_update(intel_crtc); - has_connectors = !!drm_atomic_connectors_for_crtc(state, crtc); - if (has_connectors != crtc_state->enable) - crtc_state->enable = - crtc_state->active = has_connectors; + if (modeset) + intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET); } - ret = intel_modeset_setup_plane_state(state, set->crtc, set->mode, - set->fb, set->x, set->y); - if (ret) - return ret; + /* FIXME: add subpixel order */ - crtc_state = drm_atomic_get_crtc_state(state, set->crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); + drm_atomic_helper_wait_for_vblanks(dev, state); - if (set->mode) - drm_mode_copy(&crtc_state->mode, set->mode); + mutex_lock(&dev->struct_mutex); + drm_atomic_helper_cleanup_planes(dev, state); + mutex_unlock(&dev->struct_mutex); - if (set->num_connectors) - crtc_state->active = true; + if (any_ms) + intel_modeset_check_state(dev, state); + + drm_atomic_state_free(state); return 0; } -static int intel_crtc_set_config(struct drm_mode_set *set) +void intel_crtc_restore_mode(struct drm_crtc *crtc) { - struct drm_device *dev; - struct drm_atomic_state *state = NULL; - struct intel_crtc_state *pipe_config; + struct drm_device *dev = crtc->dev; + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; int ret; - BUG_ON(!set); - BUG_ON(!set->crtc); - BUG_ON(!set->crtc->helper_private); - - /* Enforce sane interface api - has been abused by the fb helper. */ - BUG_ON(!set->mode && set->fb); - BUG_ON(set->fb && set->num_connectors == 0); - - if (set->fb) { - DRM_DEBUG_KMS("[CRTC:%d] [FB:%d] #connectors=%d (x y) (%i %i)\n", - set->crtc->base.id, set->fb->base.id, - (int)set->num_connectors, set->x, set->y); - } else { - DRM_DEBUG_KMS("[CRTC:%d] [NOFB]\n", set->crtc->base.id); - } - - dev = set->crtc->dev; - state = drm_atomic_state_alloc(dev); - if (!state) - return -ENOMEM; + if (!state) { + DRM_DEBUG_KMS("[CRTC:%d] crtc restore failed, out of memory", + crtc->base.id); + return; + } - state->acquire_ctx = dev->mode_config.acquire_ctx; + state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc); - ret = intel_modeset_stage_output_state(dev, set, state); - if (ret) - goto out; +retry: + crtc_state = drm_atomic_get_crtc_state(state, crtc); + ret = PTR_ERR_OR_ZERO(crtc_state); + if (!ret) { + if (!crtc_state->active) + goto out; - pipe_config = intel_modeset_compute_config(set->crtc, state); - if (IS_ERR(pipe_config)) { - ret = PTR_ERR(pipe_config); - goto out; + crtc_state->mode_changed = true; + ret = drm_atomic_commit(state); } - intel_update_pipe_size(to_intel_crtc(set->crtc)); - - ret = intel_set_mode_with_config(set->crtc, pipe_config, true); - - if (ret) { - DRM_DEBUG_KMS("failed to set mode on [CRTC:%d], err = %d\n", - set->crtc->base.id, ret); + if (ret == -EDEADLK) { + drm_atomic_state_clear(state); + drm_modeset_backoff(state->acquire_ctx); + goto retry; } -out: if (ret) +out: drm_atomic_state_free(state); - return ret; } +#undef for_each_intel_crtc_masked + static const struct drm_crtc_funcs intel_crtc_funcs = { .gamma_set = intel_crtc_gamma_set, - .set_config = intel_crtc_set_config, + .set_config = drm_atomic_helper_set_config, .destroy = intel_crtc_destroy, .page_flip = intel_crtc_page_flip, .atomic_duplicate_state = intel_crtc_duplicate_state, @@ -13059,28 +13635,6 @@ static void intel_shared_dpll_init(struct drm_device *dev) } /** - * intel_wm_need_update - Check whether watermarks need updating - * @plane: drm plane - * @state: new plane state - * - * Check current plane state versus the new one to determine whether - * watermarks need to be recalculated. - * - * Returns true or false. - */ -bool intel_wm_need_update(struct drm_plane *plane, - struct drm_plane_state *state) -{ - /* Update watermarks on tiling changes. */ - if (!plane->state->fb || !state->fb || - plane->state->fb->modifier[0] != state->fb->modifier[0] || - plane->state->rotation != state->rotation) - return true; - - return false; -} - -/** * intel_prepare_plane_fb - Prepare fb for usage on plane * @plane: drm plane to prepare for * @fb: framebuffer to prepare for presentation @@ -13090,52 +13644,81 @@ bool intel_wm_need_update(struct drm_plane *plane, * bits. Some older platforms need special physical address handling for * cursor planes. * + * Must be called with struct_mutex held. + * * Returns 0 on success, negative error code on failure. */ int intel_prepare_plane_fb(struct drm_plane *plane, - struct drm_framebuffer *fb, const struct drm_plane_state *new_state) { struct drm_device *dev = plane->dev; + struct drm_framebuffer *fb = new_state->fb; struct intel_plane *intel_plane = to_intel_plane(plane); - enum pipe pipe = intel_plane->pipe; struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb); - unsigned frontbuffer_bits = 0; + struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); int ret = 0; - if (!obj) + if (!obj && !old_obj) return 0; - switch (plane->type) { - case DRM_PLANE_TYPE_PRIMARY: - frontbuffer_bits = INTEL_FRONTBUFFER_PRIMARY(pipe); - break; - case DRM_PLANE_TYPE_CURSOR: - frontbuffer_bits = INTEL_FRONTBUFFER_CURSOR(pipe); - break; - case DRM_PLANE_TYPE_OVERLAY: - frontbuffer_bits = INTEL_FRONTBUFFER_SPRITE(pipe); - break; + if (old_obj) { + struct drm_crtc_state *crtc_state = + drm_atomic_get_existing_crtc_state(new_state->state, plane->state->crtc); + + /* Big Hammer, we also need to ensure that any pending + * MI_WAIT_FOR_EVENT inside a user batch buffer on the + * current scanout is retired before unpinning the old + * framebuffer. Note that we rely on userspace rendering + * into the buffer attached to the pipe they are waiting + * on. If not, userspace generates a GPU hang with IPEHR + * point to the MI_WAIT_FOR_EVENT. + * + * This should only fail upon a hung GPU, in which case we + * can safely continue. + */ + if (needs_modeset(crtc_state)) + ret = i915_gem_object_wait_rendering(old_obj, true); + + /* Swallow -EIO errors to allow updates during hw lockup. */ + if (ret && ret != -EIO) + return ret; } - mutex_lock(&dev->struct_mutex); + /* For framebuffer backed by dmabuf, wait for fence */ + if (obj && obj->base.dma_buf) { + ret = reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv, + false, true, + MAX_SCHEDULE_TIMEOUT); + if (ret == -ERESTARTSYS) + return ret; + + WARN_ON(ret < 0); + } - if (plane->type == DRM_PLANE_TYPE_CURSOR && + if (!obj) { + ret = 0; + } else if (plane->type == DRM_PLANE_TYPE_CURSOR && INTEL_INFO(dev)->cursor_needs_physical) { int align = IS_I830(dev) ? 16 * 1024 : 256; ret = i915_gem_object_attach_phys(obj, align); if (ret) DRM_DEBUG_KMS("failed to attach phys object\n"); } else { - ret = intel_pin_and_fence_fb_obj(plane, fb, new_state, NULL); + ret = intel_pin_and_fence_fb_obj(plane, fb, new_state); } - if (ret == 0) - i915_gem_track_fb(old_obj, obj, frontbuffer_bits); + if (ret == 0) { + if (obj) { + struct intel_plane_state *plane_state = + to_intel_plane_state(new_state); - mutex_unlock(&dev->struct_mutex); + i915_gem_request_assign(&plane_state->wait_req, + obj->last_write_req); + } + + i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit); + } return ret; } @@ -13146,24 +13729,35 @@ intel_prepare_plane_fb(struct drm_plane *plane, * @fb: old framebuffer that was on plane * * Cleans up a framebuffer that has just been removed from a plane. + * + * Must be called with struct_mutex held. */ void intel_cleanup_plane_fb(struct drm_plane *plane, - struct drm_framebuffer *fb, const struct drm_plane_state *old_state) { struct drm_device *dev = plane->dev; - struct drm_i915_gem_object *obj = intel_fb_obj(fb); + struct intel_plane *intel_plane = to_intel_plane(plane); + struct intel_plane_state *old_intel_state; + struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb); + struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb); + + old_intel_state = to_intel_plane_state(old_state); - if (WARN_ON(!obj)) + if (!obj && !old_obj) return; - if (plane->type != DRM_PLANE_TYPE_CURSOR || - !INTEL_INFO(dev)->cursor_needs_physical) { - mutex_lock(&dev->struct_mutex); - intel_unpin_fb_obj(fb, old_state); - mutex_unlock(&dev->struct_mutex); - } + if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR || + !INTEL_INFO(dev)->cursor_needs_physical)) + intel_unpin_fb_obj(old_state->fb, old_state); + + /* prepare_fb aborted? */ + if ((old_obj && (old_obj->frontbuffer_bits & intel_plane->frontbuffer_bit)) || + (obj && !(obj->frontbuffer_bits & intel_plane->frontbuffer_bit))) + i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit); + + i915_gem_request_assign(&old_intel_state->wait_req, NULL); + } int @@ -13180,9 +13774,9 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state dev = intel_crtc->base.dev; dev_priv = dev->dev_private; crtc_clock = crtc_state->base.adjusted_mode.crtc_clock; - cdclk = dev_priv->display.get_display_clock_speed(dev); + cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk; - if (!crtc_clock || !cdclk) + if (WARN_ON_ONCE(!crtc_clock || cdclk < crtc_clock)) return DRM_PLANE_HELPER_NO_SCALING; /* @@ -13198,112 +13792,28 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state static int intel_check_primary_plane(struct drm_plane *plane, + struct intel_crtc_state *crtc_state, struct intel_plane_state *state) { - struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = state->base.crtc; - struct intel_crtc *intel_crtc; - struct intel_crtc_state *crtc_state; struct drm_framebuffer *fb = state->base.fb; - struct drm_rect *dest = &state->dst; - struct drm_rect *src = &state->src; - const struct drm_rect *clip = &state->clip; - bool can_position = false; - int max_scale = DRM_PLANE_HELPER_NO_SCALING; int min_scale = DRM_PLANE_HELPER_NO_SCALING; - int ret; - - crtc = crtc ? crtc : plane->crtc; - intel_crtc = to_intel_crtc(crtc); - crtc_state = state->base.state ? - intel_atomic_get_crtc_state(state->base.state, intel_crtc) : NULL; + int max_scale = DRM_PLANE_HELPER_NO_SCALING; + bool can_position = false; - if (INTEL_INFO(dev)->gen >= 9) { - /* use scaler when colorkey is not required */ - if (to_intel_plane(plane)->ckey.flags == I915_SET_COLORKEY_NONE) { - min_scale = 1; - max_scale = skl_max_scale(intel_crtc, crtc_state); - } + /* use scaler when colorkey is not required */ + if (INTEL_INFO(plane->dev)->gen >= 9 && + state->ckey.flags == I915_SET_COLORKEY_NONE) { + min_scale = 1; + max_scale = skl_max_scale(to_intel_crtc(crtc), crtc_state); can_position = true; } - ret = drm_plane_helper_check_update(plane, crtc, fb, - src, dest, clip, - min_scale, - max_scale, - can_position, true, - &state->visible); - if (ret) - return ret; - - if (crtc_state ? crtc_state->base.active : intel_crtc->active) { - struct intel_plane_state *old_state = - to_intel_plane_state(plane->state); - - intel_crtc->atomic.wait_for_flips = true; - - /* - * FBC does not work on some platforms for rotated - * planes, so disable it when rotation is not 0 and - * update it when rotation is set back to 0. - * - * FIXME: This is redundant with the fbc update done in - * the primary plane enable function except that that - * one is done too late. We eventually need to unify - * this. - */ - if (state->visible && - INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && - dev_priv->fbc.crtc == intel_crtc && - state->base.rotation != BIT(DRM_ROTATE_0)) { - intel_crtc->atomic.disable_fbc = true; - } - - if (state->visible && !old_state->visible) { - /* - * BDW signals flip done immediately if the plane - * is disabled, even if the plane enable is already - * armed to occur at the next vblank :( - */ - if (IS_BROADWELL(dev)) - intel_crtc->atomic.wait_vblank = true; - - if (crtc_state) - intel_crtc->atomic.post_enable_primary = true; - } - - /* - * FIXME: Actually if we will still have any other plane enabled - * on the pipe we could let IPS enabled still, but for - * now lets consider that when we make primary invisible - * by setting DSPCNTR to 0 on update_primary_plane function - * IPS needs to be disable. - */ - if (!state->visible || !fb) - intel_crtc->atomic.disable_ips = true; - - if (!state->visible && old_state->visible && - crtc_state && !needs_modeset(&crtc_state->base)) - intel_crtc->atomic.pre_disable_primary = true; - - intel_crtc->atomic.fb_bits |= - INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe); - - intel_crtc->atomic.update_fbc = true; - - if (intel_wm_need_update(plane, &state->base)) - intel_crtc->atomic.update_wm = true; - } - - if (INTEL_INFO(dev)->gen >= 9) { - ret = skl_update_scaler_users(intel_crtc, crtc_state, - to_intel_plane(plane), state, 0); - if (ret) - return ret; - } - - return 0; + return drm_plane_helper_check_update(plane, crtc, fb, &state->src, + &state->dst, &state->clip, + min_scale, max_scale, + can_position, true, + &state->visible); } static void @@ -13314,30 +13824,17 @@ intel_commit_primary_plane(struct drm_plane *plane, struct drm_framebuffer *fb = state->base.fb; struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc; - struct drm_rect *src = &state->src; crtc = crtc ? crtc : plane->crtc; - intel_crtc = to_intel_crtc(crtc); - - plane->fb = fb; - crtc->x = src->x1 >> 16; - crtc->y = src->y1 >> 16; - if (intel_crtc->active) { - if (state->visible) - /* FIXME: kill this fastboot hack */ - intel_update_pipe_size(intel_crtc); - - dev_priv->display.update_primary_plane(crtc, plane->fb, - crtc->x, crtc->y); - } + dev_priv->display.update_primary_plane(crtc, fb, + state->src.x1 >> 16, + state->src.y1 >> 16); } static void intel_disable_primary_plane(struct drm_plane *plane, - struct drm_crtc *crtc, - bool force) + struct drm_crtc *crtc) { struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -13345,96 +13842,36 @@ intel_disable_primary_plane(struct drm_plane *plane, dev_priv->display.update_primary_plane(crtc, NULL, 0, 0); } -static void intel_begin_crtc_commit(struct drm_crtc *crtc) +static void intel_begin_crtc_commit(struct drm_crtc *crtc, + struct drm_crtc_state *old_crtc_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_plane *intel_plane; - struct drm_plane *p; - unsigned fb_bits = 0; - - /* Track fb's for any planes being disabled */ - list_for_each_entry(p, &dev->mode_config.plane_list, head) { - intel_plane = to_intel_plane(p); - - if (intel_crtc->atomic.disabled_planes & - (1 << drm_plane_index(p))) { - switch (p->type) { - case DRM_PLANE_TYPE_PRIMARY: - fb_bits = INTEL_FRONTBUFFER_PRIMARY(intel_plane->pipe); - break; - case DRM_PLANE_TYPE_CURSOR: - fb_bits = INTEL_FRONTBUFFER_CURSOR(intel_plane->pipe); - break; - case DRM_PLANE_TYPE_OVERLAY: - fb_bits = INTEL_FRONTBUFFER_SPRITE(intel_plane->pipe); - break; - } - - mutex_lock(&dev->struct_mutex); - i915_gem_track_fb(intel_fb_obj(p->fb), NULL, fb_bits); - mutex_unlock(&dev->struct_mutex); - } - } - - if (intel_crtc->atomic.wait_for_flips) - intel_crtc_wait_for_pending_flips(crtc); - - if (intel_crtc->atomic.disable_fbc) - intel_fbc_disable(dev); - - if (intel_crtc->atomic.disable_ips) - hsw_disable_ips(intel_crtc); + struct intel_crtc_state *old_intel_state = + to_intel_crtc_state(old_crtc_state); + bool modeset = needs_modeset(crtc->state); - if (intel_crtc->atomic.pre_disable_primary) - intel_pre_disable_primary(crtc); - - if (intel_crtc->atomic.update_wm) + if (intel_crtc->atomic.update_wm_pre) intel_update_watermarks(crtc); - intel_runtime_pm_get(dev_priv); - /* Perform vblank evasion around commit operation */ - if (intel_crtc->active) - intel_crtc->atomic.evade = - intel_pipe_update_start(intel_crtc, - &intel_crtc->atomic.start_vbl_count); + intel_pipe_update_start(intel_crtc); + + if (modeset) + return; + + if (to_intel_crtc_state(crtc->state)->update_pipe) + intel_update_pipe_config(intel_crtc, old_intel_state); + else if (INTEL_INFO(dev)->gen >= 9) + skl_detach_scalers(intel_crtc); } -static void intel_finish_crtc_commit(struct drm_crtc *crtc) +static void intel_finish_crtc_commit(struct drm_crtc *crtc, + struct drm_crtc_state *old_crtc_state) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_plane *p; - - if (intel_crtc->atomic.evade) - intel_pipe_update_end(intel_crtc, - intel_crtc->atomic.start_vbl_count); - - intel_runtime_pm_put(dev_priv); - - if (intel_crtc->atomic.wait_vblank) - intel_wait_for_vblank(dev, intel_crtc->pipe); - - intel_frontbuffer_flip(dev, intel_crtc->atomic.fb_bits); - - if (intel_crtc->atomic.update_fbc) { - mutex_lock(&dev->struct_mutex); - intel_fbc_update(dev); - mutex_unlock(&dev->struct_mutex); - } - - if (intel_crtc->atomic.post_enable_primary) - intel_post_enable_primary(crtc); - - drm_for_each_legacy_plane(p, &dev->mode_config.plane_list) - if (intel_crtc->atomic.update_sprite_watermarks & drm_plane_index(p)) - intel_update_sprite_watermarks(p, crtc, 0, 0, 0, - false, false); - memset(&intel_crtc->atomic, 0, sizeof(intel_crtc->atomic)); + intel_pipe_update_end(intel_crtc); } /** @@ -13469,7 +13906,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, struct intel_plane *primary; struct intel_plane_state *state; const uint32_t *intel_primary_formats; - int num_formats; + unsigned int num_formats; primary = kzalloc(sizeof(*primary), GFP_KERNEL); if (primary == NULL) @@ -13490,10 +13927,10 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, } primary->pipe = pipe; primary->plane = pipe; + primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe); primary->check_plane = intel_check_primary_plane; primary->commit_plane = intel_commit_primary_plane; primary->disable_plane = intel_disable_primary_plane; - primary->ckey.flags = I915_SET_COLORKEY_NONE; if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) primary->plane = !pipe; @@ -13511,7 +13948,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, drm_universal_plane_init(dev, &primary->base, 0, &intel_plane_funcs, intel_primary_formats, num_formats, - DRM_PLANE_TYPE_PRIMARY); + DRM_PLANE_TYPE_PRIMARY, NULL); if (INTEL_INFO(dev)->gen >= 4) intel_create_rotation_property(dev, primary); @@ -13541,37 +13978,29 @@ void intel_create_rotation_property(struct drm_device *dev, struct intel_plane * static int intel_check_cursor_plane(struct drm_plane *plane, + struct intel_crtc_state *crtc_state, struct intel_plane_state *state) { - struct drm_crtc *crtc = state->base.crtc; - struct drm_device *dev = plane->dev; + struct drm_crtc *crtc = crtc_state->base.crtc; struct drm_framebuffer *fb = state->base.fb; - struct drm_rect *dest = &state->dst; - struct drm_rect *src = &state->src; - const struct drm_rect *clip = &state->clip; struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct intel_crtc *intel_crtc; unsigned stride; int ret; - crtc = crtc ? crtc : plane->crtc; - intel_crtc = to_intel_crtc(crtc); - - ret = drm_plane_helper_check_update(plane, crtc, fb, - src, dest, clip, + ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src, + &state->dst, &state->clip, DRM_PLANE_HELPER_NO_SCALING, DRM_PLANE_HELPER_NO_SCALING, true, true, &state->visible); if (ret) return ret; - /* if we want to turn off the cursor ignore width and height */ if (!obj) - goto finish; + return 0; /* Check for which cursor types we support */ - if (!cursor_size_ok(dev, state->base.crtc_w, state->base.crtc_h)) { + if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) { DRM_DEBUG("Cursor dimension %dx%d not supported\n", state->base.crtc_w, state->base.crtc_h); return -EINVAL; @@ -13585,34 +14014,16 @@ intel_check_cursor_plane(struct drm_plane *plane, if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) { DRM_DEBUG_KMS("cursor cannot be tiled\n"); - ret = -EINVAL; - } - -finish: - if (intel_crtc->active) { - if (plane->state->crtc_w != state->base.crtc_w) - intel_crtc->atomic.update_wm = true; - - intel_crtc->atomic.fb_bits |= - INTEL_FRONTBUFFER_CURSOR(intel_crtc->pipe); + return -EINVAL; } - return ret; + return 0; } static void intel_disable_cursor_plane(struct drm_plane *plane, - struct drm_crtc *crtc, - bool force) + struct drm_crtc *crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - - if (!force) { - plane->fb = NULL; - intel_crtc->cursor_bo = NULL; - intel_crtc->cursor_addr = 0; - } - intel_crtc_update_cursor(crtc, false); } @@ -13629,10 +14040,6 @@ intel_commit_cursor_plane(struct drm_plane *plane, crtc = crtc ? crtc : plane->crtc; intel_crtc = to_intel_crtc(crtc); - plane->fb = state->base.fb; - crtc->cursor_x = state->base.crtc_x; - crtc->cursor_y = state->base.crtc_y; - if (intel_crtc->cursor_bo == obj) goto update; @@ -13645,10 +14052,9 @@ intel_commit_cursor_plane(struct drm_plane *plane, intel_crtc->cursor_addr = addr; intel_crtc->cursor_bo = obj; -update: - if (intel_crtc->active) - intel_crtc_update_cursor(crtc, state->visible); +update: + intel_crtc_update_cursor(crtc, state->visible); } static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, @@ -13672,6 +14078,7 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, cursor->max_downscale = 1; cursor->pipe = pipe; cursor->plane = pipe; + cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe); cursor->check_plane = intel_check_cursor_plane; cursor->commit_plane = intel_commit_cursor_plane; cursor->disable_plane = intel_disable_cursor_plane; @@ -13680,7 +14087,7 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, &intel_plane_funcs, intel_cursor_formats, ARRAY_SIZE(intel_cursor_formats), - DRM_PLANE_TYPE_CURSOR); + DRM_PLANE_TYPE_CURSOR, NULL); if (INTEL_INFO(dev)->gen >= 4) { if (!dev->mode_config.rotation_property) @@ -13712,8 +14119,6 @@ static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_cr for (i = 0; i < intel_crtc->num_scalers; i++) { intel_scaler = &scaler_state->scalers[i]; intel_scaler->in_use = 0; - intel_scaler->id = i; - intel_scaler->mode = PS_SCALER_MODE_DYN; } @@ -13759,7 +14164,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) goto fail; ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary, - cursor, &intel_crtc_funcs); + cursor, &intel_crtc_funcs, NULL); if (ret) goto fail; @@ -13785,6 +14190,8 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) intel_crtc->cursor_cntl = ~0; intel_crtc->cursor_size = ~0; + intel_crtc->wm.cxsr_allowed = true; + BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL); dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; @@ -13883,7 +14290,14 @@ static bool intel_crt_present(struct drm_device *dev) if (IS_CHERRYVIEW(dev)) return false; - if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support) + if (HAS_PCH_LPT_H(dev) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED) + return false; + + /* DDI E can't be used if DDI A requires 4 lanes */ + if (HAS_DDI(dev) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) + return false; + + if (!dev_priv->vbt.int_crt_support) return false; return true; @@ -13917,10 +14331,9 @@ static void intel_setup_outputs(struct drm_device *dev) * On SKL pre-D0 the strap isn't connected, so we assume * it's there. */ - found = I915_READ(DDI_BUF_CTL_A) & DDI_INIT_DISPLAY_DETECTED; + found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED; /* WaIgnoreDDIAStrap: skl */ - if (found || - (IS_SKYLAKE(dev) && INTEL_REVID(dev) < SKL_REVID_D0)) + if (found || IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) intel_ddi_init(dev, PORT_A); /* DDI B, C and D detection is indicated by the SFUSE_STRAP @@ -13933,6 +14346,15 @@ static void intel_setup_outputs(struct drm_device *dev) intel_ddi_init(dev, PORT_C); if (found & SFUSE_STRAP_DDID_DETECTED) intel_ddi_init(dev, PORT_D); + /* + * On SKL we don't have a way to detect DDI-E so we rely on VBT. + */ + if ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) && + (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp || + dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi || + dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi)) + intel_ddi_init(dev, PORT_E); + } else if (HAS_PCH_SPLIT(dev)) { int found; dpd_is_edp = intel_dp_is_edp(dev, PORT_D); @@ -13942,7 +14364,7 @@ static void intel_setup_outputs(struct drm_device *dev) if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) { /* PCH SDVOB multiplex with HDMIB */ - found = intel_sdvo_init(dev, PCH_SDVOB, true); + found = intel_sdvo_init(dev, PCH_SDVOB, PORT_B); if (!found) intel_hdmi_init(dev, PCH_HDMIB, PORT_B); if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED)) @@ -13970,44 +14392,41 @@ static void intel_setup_outputs(struct drm_device *dev) * eDP ports. Consult the VBT as well as DP_DETECTED to * detect eDP ports. */ - if (I915_READ(VLV_DISPLAY_BASE + GEN4_HDMIB) & SDVO_DETECTED && + if (I915_READ(VLV_HDMIB) & SDVO_DETECTED && !intel_dp_is_edp(dev, PORT_B)) - intel_hdmi_init(dev, VLV_DISPLAY_BASE + GEN4_HDMIB, - PORT_B); - if (I915_READ(VLV_DISPLAY_BASE + DP_B) & DP_DETECTED || + intel_hdmi_init(dev, VLV_HDMIB, PORT_B); + if (I915_READ(VLV_DP_B) & DP_DETECTED || intel_dp_is_edp(dev, PORT_B)) - intel_dp_init(dev, VLV_DISPLAY_BASE + DP_B, PORT_B); + intel_dp_init(dev, VLV_DP_B, PORT_B); - if (I915_READ(VLV_DISPLAY_BASE + GEN4_HDMIC) & SDVO_DETECTED && + if (I915_READ(VLV_HDMIC) & SDVO_DETECTED && !intel_dp_is_edp(dev, PORT_C)) - intel_hdmi_init(dev, VLV_DISPLAY_BASE + GEN4_HDMIC, - PORT_C); - if (I915_READ(VLV_DISPLAY_BASE + DP_C) & DP_DETECTED || + intel_hdmi_init(dev, VLV_HDMIC, PORT_C); + if (I915_READ(VLV_DP_C) & DP_DETECTED || intel_dp_is_edp(dev, PORT_C)) - intel_dp_init(dev, VLV_DISPLAY_BASE + DP_C, PORT_C); + intel_dp_init(dev, VLV_DP_C, PORT_C); if (IS_CHERRYVIEW(dev)) { - if (I915_READ(VLV_DISPLAY_BASE + CHV_HDMID) & SDVO_DETECTED) - intel_hdmi_init(dev, VLV_DISPLAY_BASE + CHV_HDMID, - PORT_D); /* eDP not supported on port D, so don't check VBT */ - if (I915_READ(VLV_DISPLAY_BASE + DP_D) & DP_DETECTED) - intel_dp_init(dev, VLV_DISPLAY_BASE + DP_D, PORT_D); + if (I915_READ(CHV_HDMID) & SDVO_DETECTED) + intel_hdmi_init(dev, CHV_HDMID, PORT_D); + if (I915_READ(CHV_DP_D) & DP_DETECTED) + intel_dp_init(dev, CHV_DP_D, PORT_D); } intel_dsi_init(dev); - } else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) { + } else if (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) { bool found = false; if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) { DRM_DEBUG_KMS("probing SDVOB\n"); - found = intel_sdvo_init(dev, GEN3_SDVOB, true); - if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) { + found = intel_sdvo_init(dev, GEN3_SDVOB, PORT_B); + if (!found && IS_G4X(dev)) { DRM_DEBUG_KMS("probing HDMI on SDVOB\n"); intel_hdmi_init(dev, GEN4_HDMIB, PORT_B); } - if (!found && SUPPORTS_INTEGRATED_DP(dev)) + if (!found && IS_G4X(dev)) intel_dp_init(dev, DP_B, PORT_B); } @@ -14015,20 +14434,20 @@ static void intel_setup_outputs(struct drm_device *dev) if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) { DRM_DEBUG_KMS("probing SDVOC\n"); - found = intel_sdvo_init(dev, GEN3_SDVOC, false); + found = intel_sdvo_init(dev, GEN3_SDVOC, PORT_C); } if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) { - if (SUPPORTS_INTEGRATED_HDMI(dev)) { + if (IS_G4X(dev)) { DRM_DEBUG_KMS("probing HDMI on SDVOC\n"); intel_hdmi_init(dev, GEN4_HDMIC, PORT_C); } - if (SUPPORTS_INTEGRATED_DP(dev)) + if (IS_G4X(dev)) intel_dp_init(dev, DP_C, PORT_C); } - if (SUPPORTS_INTEGRATED_DP(dev) && + if (IS_G4X(dev) && (I915_READ(DP_D) & DP_DETECTED)) intel_dp_init(dev, DP_D, PORT_D); } else if (IS_GEN2(dev)) @@ -14070,12 +14489,35 @@ static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; + if (obj->userptr.mm) { + DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n"); + return -EINVAL; + } + return drm_gem_handle_create(file, &obj->base, handle); } +static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb, + struct drm_file *file, + unsigned flags, unsigned color, + struct drm_clip_rect *clips, + unsigned num_clips) +{ + struct drm_device *dev = fb->dev; + struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); + struct drm_i915_gem_object *obj = intel_fb->obj; + + mutex_lock(&dev->struct_mutex); + intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB); + mutex_unlock(&dev->struct_mutex); + + return 0; +} + static const struct drm_framebuffer_funcs intel_fb_funcs = { .destroy = intel_user_framebuffer_destroy, .create_handle = intel_user_framebuffer_create_handle, + .dirty = intel_user_framebuffer_dirty, }; static @@ -14258,19 +14700,25 @@ static int intel_framebuffer_init(struct drm_device *dev, static struct drm_framebuffer * intel_user_framebuffer_create(struct drm_device *dev, struct drm_file *filp, - struct drm_mode_fb_cmd2 *mode_cmd) + const struct drm_mode_fb_cmd2 *user_mode_cmd) { + struct drm_framebuffer *fb; struct drm_i915_gem_object *obj; + struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd; obj = to_intel_bo(drm_gem_object_lookup(dev, filp, - mode_cmd->handles[0])); + mode_cmd.handles[0])); if (&obj->base == NULL) return ERR_PTR(-ENOENT); - return intel_framebuffer_create(dev, mode_cmd, obj); + fb = intel_framebuffer_create(dev, &mode_cmd, obj); + if (IS_ERR(fb)) + drm_gem_object_unreference_unlocked(&obj->base); + + return fb; } -#ifndef CONFIG_DRM_I915_FBDEV +#ifndef CONFIG_DRM_FBDEV_EMULATION static inline void intel_fbdev_output_poll_changed(struct drm_device *dev) { } @@ -14281,6 +14729,8 @@ static const struct drm_mode_config_funcs intel_mode_funcs = { .output_poll_changed = intel_fbdev_output_poll_changed, .atomic_check = intel_atomic_check, .atomic_commit = intel_atomic_commit, + .atomic_state_alloc = intel_atomic_state_alloc, + .atomic_state_clear = intel_atomic_state_clear, }; /* Set up chip specific display functions */ @@ -14307,7 +14757,6 @@ static void intel_init_display(struct drm_device *dev) haswell_crtc_compute_clock; dev_priv->display.crtc_enable = haswell_crtc_enable; dev_priv->display.crtc_disable = haswell_crtc_disable; - dev_priv->display.off = ironlake_crtc_off; dev_priv->display.update_primary_plane = skylake_update_primary_plane; } else if (HAS_DDI(dev)) { @@ -14318,7 +14767,6 @@ static void intel_init_display(struct drm_device *dev) haswell_crtc_compute_clock; dev_priv->display.crtc_enable = haswell_crtc_enable; dev_priv->display.crtc_disable = haswell_crtc_disable; - dev_priv->display.off = ironlake_crtc_off; dev_priv->display.update_primary_plane = ironlake_update_primary_plane; } else if (HAS_PCH_SPLIT(dev)) { @@ -14329,7 +14777,6 @@ static void intel_init_display(struct drm_device *dev) ironlake_crtc_compute_clock; dev_priv->display.crtc_enable = ironlake_crtc_enable; dev_priv->display.crtc_disable = ironlake_crtc_disable; - dev_priv->display.off = ironlake_crtc_off; dev_priv->display.update_primary_plane = ironlake_update_primary_plane; } else if (IS_VALLEYVIEW(dev)) { @@ -14339,7 +14786,6 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock; dev_priv->display.crtc_enable = valleyview_crtc_enable; dev_priv->display.crtc_disable = i9xx_crtc_disable; - dev_priv->display.off = i9xx_crtc_off; dev_priv->display.update_primary_plane = i9xx_update_primary_plane; } else { @@ -14349,15 +14795,17 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock; dev_priv->display.crtc_enable = i9xx_crtc_enable; dev_priv->display.crtc_disable = i9xx_crtc_disable; - dev_priv->display.off = i9xx_crtc_off; dev_priv->display.update_primary_plane = i9xx_update_primary_plane; } /* Returns the core display clock speed */ - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) dev_priv->display.get_display_clock_speed = skylake_get_display_clock_speed; + else if (IS_BROXTON(dev)) + dev_priv->display.get_display_clock_speed = + broxton_get_display_clock_speed; else if (IS_BROADWELL(dev)) dev_priv->display.get_display_clock_speed = broadwell_get_display_clock_speed; @@ -14371,18 +14819,27 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.get_display_clock_speed = ilk_get_display_clock_speed; else if (IS_I945G(dev) || IS_BROADWATER(dev) || - IS_GEN6(dev) || IS_IVYBRIDGE(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev))) + IS_GEN6(dev) || IS_IVYBRIDGE(dev)) dev_priv->display.get_display_clock_speed = i945_get_display_clock_speed; + else if (IS_GM45(dev)) + dev_priv->display.get_display_clock_speed = + gm45_get_display_clock_speed; + else if (IS_CRESTLINE(dev)) + dev_priv->display.get_display_clock_speed = + i965gm_get_display_clock_speed; + else if (IS_PINEVIEW(dev)) + dev_priv->display.get_display_clock_speed = + pnv_get_display_clock_speed; + else if (IS_G33(dev) || IS_G4X(dev)) + dev_priv->display.get_display_clock_speed = + g33_get_display_clock_speed; else if (IS_I915G(dev)) dev_priv->display.get_display_clock_speed = i915_get_display_clock_speed; else if (IS_I945GM(dev) || IS_845G(dev)) dev_priv->display.get_display_clock_speed = i9xx_misc_get_display_clock_speed; - else if (IS_PINEVIEW(dev)) - dev_priv->display.get_display_clock_speed = - pnv_get_display_clock_speed; else if (IS_I915GM(dev)) dev_priv->display.get_display_clock_speed = i915gm_get_display_clock_speed; @@ -14391,10 +14848,12 @@ static void intel_init_display(struct drm_device *dev) i865_get_display_clock_speed; else if (IS_I85X(dev)) dev_priv->display.get_display_clock_speed = - i855_get_display_clock_speed; - else /* 852, 830 */ + i85x_get_display_clock_speed; + else { /* 830 */ + WARN(!IS_I830(dev), "Unknown platform. Assuming 133 MHz CDCLK\n"); dev_priv->display.get_display_clock_speed = i830_get_display_clock_speed; + } if (IS_GEN5(dev)) { dev_priv->display.fdi_link_train = ironlake_fdi_link_train; @@ -14405,12 +14864,22 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { dev_priv->display.fdi_link_train = hsw_fdi_link_train; + if (IS_BROADWELL(dev)) { + dev_priv->display.modeset_commit_cdclk = + broadwell_modeset_commit_cdclk; + dev_priv->display.modeset_calc_cdclk = + broadwell_modeset_calc_cdclk; + } } else if (IS_VALLEYVIEW(dev)) { - dev_priv->display.modeset_global_resources = - valleyview_modeset_global_resources; + dev_priv->display.modeset_commit_cdclk = + valleyview_modeset_commit_cdclk; + dev_priv->display.modeset_calc_cdclk = + valleyview_modeset_calc_cdclk; } else if (IS_BROXTON(dev)) { - dev_priv->display.modeset_global_resources = - broxton_modeset_global_resources; + dev_priv->display.modeset_commit_cdclk = + broxton_modeset_commit_cdclk; + dev_priv->display.modeset_calc_cdclk = + broxton_modeset_calc_cdclk; } switch (INTEL_INFO(dev)->gen) { @@ -14441,8 +14910,6 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.queue_flip = intel_default_queue_flip; } - intel_panel_init_backlight_funcs(dev); - mutex_init(&dev_priv->pps_mutex); } @@ -14577,6 +15044,9 @@ static struct intel_quirk intel_quirks[] = { /* Apple Macbook 2,1 (Core 2 T7400) */ { 0x27a2, 0x8086, 0x7270, quirk_backlight_present }, + /* Apple Macbook 4,1 */ + { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present }, + /* Toshiba CB35 Chromebook (Celeron 2955U) */ { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present }, @@ -14585,6 +15055,9 @@ static struct intel_quirk intel_quirks[] = { /* Dell Chromebook 11 */ { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present }, + + /* Dell Chromebook 11 (2015 version) */ + { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present }, }; static void intel_init_quirks(struct drm_device *dev) @@ -14613,7 +15086,7 @@ static void i915_disable_vga(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u8 sr1; - u32 vga_reg = i915_vgacntrl_reg(dev); + i915_reg_t vga_reg = i915_vgacntrl_reg(dev); /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */ vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO); @@ -14629,13 +15102,9 @@ static void i915_disable_vga(struct drm_device *dev) void intel_modeset_init_hw(struct drm_device *dev) { + intel_update_cdclk(dev); intel_prepare_ddi(dev); - - if (IS_VALLEYVIEW(dev)) - vlv_update_cdclk(dev); - intel_init_clock_gating(dev); - intel_enable_gt_powersave(dev); } @@ -14665,6 +15134,24 @@ void intel_modeset_init(struct drm_device *dev) if (INTEL_INFO(dev)->num_pipes == 0) return; + /* + * There may be no VBT; and if the BIOS enabled SSC we can + * just keep using it to avoid unnecessary flicker. Whereas if the + * BIOS isn't using it, don't assume it will work even if the VBT + * indicates as much. + */ + if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) { + bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) & + DREF_SSC1_ENABLE); + + if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) { + DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n", + bios_lvds_use_ssc ? "en" : "dis", + dev_priv->vbt.lvds_use_ssc ? "en" : "dis"); + dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc; + } + } + intel_init_display(dev); intel_init_audio(dev); @@ -14706,7 +15193,8 @@ void intel_modeset_init(struct drm_device *dev) } } - intel_init_dpio(dev); + intel_update_czclk(dev_priv); + intel_update_cdclk(dev); intel_shared_dpll_init(dev); @@ -14714,14 +15202,13 @@ void intel_modeset_init(struct drm_device *dev) i915_disable_vga(dev); intel_setup_outputs(dev); - /* Just in case the BIOS is doing something questionable. */ - intel_fbc_disable(dev); - drm_modeset_lock_all(dev); - intel_modeset_setup_hw_state(dev, false); + intel_modeset_setup_hw_state(dev); drm_modeset_unlock_all(dev); for_each_intel_crtc(dev, crtc) { + struct intel_initial_plane_config plane_config = {}; + if (!crtc->active) continue; @@ -14732,15 +15219,14 @@ void intel_modeset_init(struct drm_device *dev) * can even allow for smooth boot transitions if the BIOS * fb is large enough for the active pipe configuration. */ - if (dev_priv->display.get_initial_plane_config) { - dev_priv->display.get_initial_plane_config(crtc, - &crtc->plane_config); - /* - * If the fb is shared between multiple heads, we'll - * just get the first one. - */ - intel_find_initial_plane_obj(crtc, &crtc->plane_config); - } + dev_priv->display.get_initial_plane_config(crtc, + &plane_config); + + /* + * If the fb is shared between multiple heads, we'll + * just get the first one. + */ + intel_find_initial_plane_obj(crtc, &plane_config); } } @@ -14773,13 +15259,12 @@ intel_check_plane_mapping(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg, val; + u32 val; if (INTEL_INFO(dev)->num_pipes == 1) return true; - reg = DSPCNTR(!crtc->plane); - val = I915_READ(reg); + val = I915_READ(DSPCNTR(!crtc->plane)); if ((val & DISPLAY_PLANE_ENABLE) && (!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe)) @@ -14788,28 +15273,46 @@ intel_check_plane_mapping(struct intel_crtc *crtc) return true; } +static bool intel_crtc_has_encoders(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct intel_encoder *encoder; + + for_each_encoder_on_crtc(dev, &crtc->base, encoder) + return true; + + return false; +} + static void intel_sanitize_crtc(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg; + i915_reg_t reg = PIPECONF(crtc->config->cpu_transcoder); /* Clear any frame start delays used for debugging left by the BIOS */ - reg = PIPECONF(crtc->config->cpu_transcoder); I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK); /* restore vblank interrupts to correct state */ drm_crtc_vblank_reset(&crtc->base); if (crtc->active) { - update_scanline_offset(crtc); + struct intel_plane *plane; + drm_crtc_vblank_on(&crtc->base); + + /* Disable everything but the primary plane */ + for_each_intel_plane_on_crtc(dev, crtc, plane) { + if (plane->base.type == DRM_PLANE_TYPE_PRIMARY) + continue; + + plane->disable_plane(&plane->base, &crtc->base); + } } /* We need to sanitize the plane -> pipe mapping first because this will * disable the crtc (and hence change the state) if it is wrong. Note * that gen4+ has a fixed plane -> pipe mapping. */ if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) { - struct intel_connector *connector; bool plane; DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n", @@ -14821,30 +15324,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) plane = crtc->plane; to_intel_plane_state(crtc->base.primary->state)->visible = true; crtc->plane = !plane; - intel_crtc_disable_planes(&crtc->base); - dev_priv->display.crtc_disable(&crtc->base); + intel_crtc_disable_noatomic(&crtc->base); crtc->plane = plane; - - /* ... and break all links. */ - for_each_intel_connector(dev, connector) { - if (connector->encoder->base.crtc != &crtc->base) - continue; - - connector->base.dpms = DRM_MODE_DPMS_OFF; - connector->base.encoder = NULL; - } - /* multiple connectors may have the same encoder: - * handle them and break crtc link separately */ - for_each_intel_connector(dev, connector) - if (connector->encoder->base.crtc == &crtc->base) { - connector->encoder->base.crtc = NULL; - connector->encoder->connectors_active = false; - } - - WARN_ON(crtc->active); - crtc->base.state->enable = false; - crtc->base.state->active = false; - crtc->base.enabled = false; } if (dev_priv->quirks & QUIRK_PIPEA_FORCE && @@ -14858,20 +15339,22 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) /* Adjust the state of the output pipe according to whether we * have active connectors/encoders. */ - intel_crtc_update_dpms(&crtc->base); + if (!intel_crtc_has_encoders(crtc)) + intel_crtc_disable_noatomic(&crtc->base); - if (crtc->active != crtc->base.state->enable) { + if (crtc->active != crtc->base.state->active) { struct intel_encoder *encoder; /* This can happen either due to bugs in the get_hw_state - * functions or because the pipe is force-enabled due to the + * functions or because of calls to intel_crtc_disable_noatomic, + * or because the pipe is force-enabled due to the * pipe A quirk. */ DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was %s, now %s\n", crtc->base.base.id, crtc->base.state->enable ? "enabled" : "disabled", crtc->active ? "enabled" : "disabled"); - crtc->base.state->enable = crtc->active; + WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, NULL) < 0); crtc->base.state->active = crtc->active; crtc->base.enabled = crtc->active; @@ -14882,10 +15365,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) * actually up, hence no need to break them. */ WARN_ON(crtc->active); - for_each_encoder_on_crtc(dev, &crtc->base, encoder) { - WARN_ON(encoder->connectors_active); + for_each_encoder_on_crtc(dev, &crtc->base, encoder) encoder->base.crtc = NULL; - } } if (crtc->active || HAS_GMCH_DISPLAY(dev)) { @@ -14911,6 +15392,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) { struct intel_connector *connector; struct drm_device *dev = encoder->base.dev; + bool active = false; /* We need to check both for a crtc link (meaning that the * encoder is active and trying to read from a pipe) and the @@ -14918,7 +15400,15 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) bool has_active_crtc = encoder->base.crtc && to_intel_crtc(encoder->base.crtc)->active; - if (encoder->connectors_active && !has_active_crtc) { + for_each_intel_connector(dev, connector) { + if (connector->base.encoder != &encoder->base) + continue; + + active = true; + break; + } + + if (active && !has_active_crtc) { DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n", encoder->base.base.id, encoder->base.name); @@ -14935,7 +15425,6 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) encoder->post_disable(encoder); } encoder->base.crtc = NULL; - encoder->connectors_active = false; /* Inconsistent output/port/pipe state happens presumably due to * a bug in one of the get_hw_state functions. Or someplace else @@ -14955,7 +15444,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) void i915_redisable_vga_power_on(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 vga_reg = i915_vgacntrl_reg(dev); + i915_reg_t vga_reg = i915_vgacntrl_reg(dev); if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n"); @@ -14980,14 +15469,25 @@ void i915_redisable_vga(struct drm_device *dev) i915_redisable_vga_power_on(dev); } -static bool primary_get_hw_state(struct intel_crtc *crtc) +static bool primary_get_hw_state(struct intel_plane *plane) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - if (!crtc->active) - return false; + return I915_READ(DSPCNTR(plane->plane)) & DISPLAY_PLANE_ENABLE; +} - return I915_READ(DSPCNTR(crtc->plane)) & DISPLAY_PLANE_ENABLE; +/* FIXME read out full plane state for all planes */ +static void readout_plane_state(struct intel_crtc *crtc) +{ + struct drm_plane *primary = crtc->base.primary; + struct intel_plane_state *plane_state = + to_intel_plane_state(primary->state); + + plane_state->visible = crtc->active && + primary_get_hw_state(to_intel_plane(primary)); + + if (plane_state->visible) + crtc->base.state->plane_mask |= 1 << drm_plane_index(primary); } static void intel_modeset_readout_hw_state(struct drm_device *dev) @@ -15000,23 +15500,17 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) int i; for_each_intel_crtc(dev, crtc) { - struct drm_plane *primary = crtc->base.primary; - struct intel_plane_state *plane_state; - + __drm_atomic_helper_crtc_destroy_state(&crtc->base, crtc->base.state); memset(crtc->config, 0, sizeof(*crtc->config)); crtc->config->base.crtc = &crtc->base; - crtc->config->quirks |= PIPE_CONFIG_QUIRK_INHERITED_MODE; - crtc->active = dev_priv->display.get_pipe_config(crtc, crtc->config); - crtc->base.state->enable = crtc->active; crtc->base.state->active = crtc->active; crtc->base.enabled = crtc->active; - plane_state = to_intel_plane_state(primary->state); - plane_state->visible = primary_get_hw_state(crtc); + readout_plane_state(crtc); DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n", crtc->base.base.id, @@ -15055,7 +15549,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) encoder->base.crtc = NULL; } - encoder->connectors_active = false; DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n", encoder->base.base.id, encoder->base.name, @@ -15066,7 +15559,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) for_each_intel_connector(dev, connector) { if (connector->get_hw_state(connector)) { connector->base.dpms = DRM_MODE_DPMS_ON; - connector->encoder->connectors_active = true; connector->base.encoder = &connector->encoder->base; } else { connector->base.dpms = DRM_MODE_DPMS_OFF; @@ -15077,12 +15569,46 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) connector->base.name, connector->base.encoder ? "enabled" : "disabled"); } + + for_each_intel_crtc(dev, crtc) { + crtc->base.hwmode = crtc->config->base.adjusted_mode; + + memset(&crtc->base.mode, 0, sizeof(crtc->base.mode)); + if (crtc->base.state->active) { + intel_mode_from_pipe_config(&crtc->base.mode, crtc->config); + intel_mode_from_pipe_config(&crtc->base.state->adjusted_mode, crtc->config); + WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode)); + + /* + * The initial mode needs to be set in order to keep + * the atomic core happy. It wants a valid mode if the + * crtc's enabled, so we do the above call. + * + * At this point some state updated by the connectors + * in their ->detect() callback has not run yet, so + * no recalculation can be done yet. + * + * Even if we could do a recalculation and modeset + * right now it would cause a double modeset if + * fbdev or userspace chooses a different initial mode. + * + * If that happens, someone indicated they wanted a + * mode change, which means it's safe to do a full + * recalculation. + */ + crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED; + + drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode); + update_scanline_offset(crtc); + } + } } -/* Scan out the current hw modeset state, sanitizes it and maps it into the drm - * and i915 state tracking structures. */ -void intel_modeset_setup_hw_state(struct drm_device *dev, - bool force_restore) +/* Scan out the current hw modeset state, + * and sanitizes it to the current state + */ +static void +intel_modeset_setup_hw_state(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; @@ -15092,21 +15618,6 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, intel_modeset_readout_hw_state(dev); - /* - * Now that we have the config, copy it to each CRTC struct - * Note that this could go away if we move to using crtc_config - * checking everywhere. - */ - for_each_intel_crtc(dev, crtc) { - if (crtc->active && i915.fastboot) { - intel_mode_from_pipe_config(&crtc->base.mode, - crtc->config); - DRM_DEBUG_KMS("[CRTC:%d] found active mode: ", - crtc->base.base.id); - drm_mode_debug_printmodeline(&crtc->base.mode); - } - } - /* HW state is read out, now we need to sanitize this mess. */ for_each_intel_encoder(dev, encoder) { intel_sanitize_encoder(encoder); @@ -15133,34 +15644,77 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, pll->on = false; } - if (IS_GEN9(dev)) + if (IS_VALLEYVIEW(dev)) + vlv_wm_get_hw_state(dev); + else if (IS_GEN9(dev)) skl_wm_get_hw_state(dev); else if (HAS_PCH_SPLIT(dev)) ilk_wm_get_hw_state(dev); - if (force_restore) { - i915_redisable_vga(dev); + for_each_intel_crtc(dev, crtc) { + unsigned long put_domains; - /* - * We need to use raw interfaces for restoring state to avoid - * checking (bogus) intermediate states. - */ - for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = - dev_priv->pipe_to_crtc_mapping[pipe]; + put_domains = modeset_get_crtc_power_domains(&crtc->base); + if (WARN_ON(put_domains)) + modeset_put_power_domains(dev_priv, put_domains); + } + intel_display_set_init_power(dev_priv, false); +} - intel_crtc_restore_mode(crtc); - } - } else { - intel_modeset_update_staged_output_state(dev); +void intel_display_resume(struct drm_device *dev) +{ + struct drm_atomic_state *state = drm_atomic_state_alloc(dev); + struct intel_connector *conn; + struct intel_plane *plane; + struct drm_crtc *crtc; + int ret; + + if (!state) + return; + + state->acquire_ctx = dev->mode_config.acquire_ctx; + + /* preserve complete old state, including dpll */ + intel_atomic_get_shared_dpll_state(state); + + for_each_crtc(dev, crtc) { + struct drm_crtc_state *crtc_state = + drm_atomic_get_crtc_state(state, crtc); + + ret = PTR_ERR_OR_ZERO(crtc_state); + if (ret) + goto err; + + /* force a restore */ + crtc_state->mode_changed = true; } - intel_modeset_check_state(dev); + for_each_intel_plane(dev, plane) { + ret = PTR_ERR_OR_ZERO(drm_atomic_get_plane_state(state, &plane->base)); + if (ret) + goto err; + } + + for_each_intel_connector(dev, conn) { + ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(state, &conn->base)); + if (ret) + goto err; + } + + intel_modeset_setup_hw_state(dev); + + i915_redisable_vga(dev); + ret = drm_atomic_commit(state); + if (!ret) + return; + +err: + DRM_ERROR("Restoring old state failed with %i\n", ret); + drm_atomic_state_free(state); } void intel_modeset_gem_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *c; struct drm_i915_gem_object *obj; int ret; @@ -15169,16 +15723,6 @@ void intel_modeset_gem_init(struct drm_device *dev) intel_init_gt_powersave(dev); mutex_unlock(&dev->struct_mutex); - /* - * There may be no VBT; and if the BIOS enabled SSC we can - * just keep using it to avoid unnecessary flicker. Whereas if the - * BIOS isn't using it, don't assume it will work even if the VBT - * indicates as much. - */ - if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) - dev_priv->vbt.lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) & - DREF_SSC1_ENABLE); - intel_modeset_init_hw(dev); intel_setup_overlay(dev); @@ -15196,15 +15740,16 @@ void intel_modeset_gem_init(struct drm_device *dev) mutex_lock(&dev->struct_mutex); ret = intel_pin_and_fence_fb_obj(c->primary, c->primary->fb, - c->primary->state, - NULL); + c->primary->state); mutex_unlock(&dev->struct_mutex); if (ret) { DRM_ERROR("failed to pin boot fb on pipe %d\n", to_intel_crtc(c)->pipe); drm_framebuffer_unreference(c->primary->fb); c->primary->fb = NULL; + c->primary->crtc = c->primary->state->crtc = NULL; update_state_fb(c->primary); + c->state->plane_mask &= ~(1 << drm_plane_index(c->primary)); } } @@ -15241,13 +15786,9 @@ void intel_modeset_cleanup(struct drm_device *dev) */ drm_kms_helper_poll_fini(dev); - mutex_lock(&dev->struct_mutex); - intel_unregister_dsm_handler(); - intel_fbc_disable(dev); - - mutex_unlock(&dev->struct_mutex); + intel_fbc_disable(dev_priv); /* flush any delayed tasks or pending work */ flush_scheduled_work(); diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 1df0e1fe235f..0f0573aa1b0d 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -48,28 +48,28 @@ #define INTEL_DP_RESOLUTION_FAILSAFE (3 << INTEL_DP_RESOLUTION_SHIFT_MASK) struct dp_link_dpll { - int link_bw; + int clock; struct dpll dpll; }; static const struct dp_link_dpll gen4_dpll[] = { - { DP_LINK_BW_1_62, + { 162000, { .p1 = 2, .p2 = 10, .n = 2, .m1 = 23, .m2 = 8 } }, - { DP_LINK_BW_2_7, + { 270000, { .p1 = 1, .p2 = 10, .n = 1, .m1 = 14, .m2 = 2 } } }; static const struct dp_link_dpll pch_dpll[] = { - { DP_LINK_BW_1_62, + { 162000, { .p1 = 2, .p2 = 10, .n = 1, .m1 = 12, .m2 = 9 } }, - { DP_LINK_BW_2_7, + { 270000, { .p1 = 1, .p2 = 10, .n = 2, .m1 = 14, .m2 = 8 } } }; static const struct dp_link_dpll vlv_dpll[] = { - { DP_LINK_BW_1_62, + { 162000, { .p1 = 3, .p2 = 2, .n = 5, .m1 = 3, .m2 = 81 } }, - { DP_LINK_BW_2_7, + { 270000, { .p1 = 2, .p2 = 2, .n = 1, .m1 = 2, .m2 = 27 } } }; @@ -83,14 +83,16 @@ static const struct dp_link_dpll chv_dpll[] = { * m2 is stored in fixed point format using formula below * (m2_int << 22) | m2_fraction */ - { DP_LINK_BW_1_62, /* m2_int = 32, m2_fraction = 1677722 */ + { 162000, /* m2_int = 32, m2_fraction = 1677722 */ { .p1 = 4, .p2 = 2, .n = 1, .m1 = 2, .m2 = 0x819999a } }, - { DP_LINK_BW_2_7, /* m2_int = 27, m2_fraction = 0 */ + { 270000, /* m2_int = 27, m2_fraction = 0 */ { .p1 = 4, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } }, - { DP_LINK_BW_5_4, /* m2_int = 27, m2_fraction = 0 */ + { 540000, /* m2_int = 27, m2_fraction = 0 */ { .p1 = 2, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } } }; +static const int bxt_rates[] = { 162000, 216000, 243000, 270000, + 324000, 432000, 540000 }; static const int skl_rates[] = { 162000, 216000, 270000, 324000, 432000, 540000 }; static const int default_rates[] = { 162000, 270000, 540000 }; @@ -128,6 +130,11 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp); static void vlv_steal_power_sequencer(struct drm_device *dev, enum pipe pipe); +static unsigned int intel_dp_unused_lane_mask(int lane_count) +{ + return ~((1 << lane_count) - 1) & 0xf; +} + static int intel_dp_max_link_bw(struct intel_dp *intel_dp) { @@ -251,40 +258,6 @@ static void intel_dp_unpack_aux(uint32_t src, uint8_t *dst, int dst_bytes) dst[i] = src >> ((3-i) * 8); } -/* hrawclock is 1/4 the FSB frequency */ -static int -intel_hrawclk(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t clkcfg; - - /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */ - if (IS_VALLEYVIEW(dev)) - return 200; - - clkcfg = I915_READ(CLKCFG); - switch (clkcfg & CLKCFG_FSB_MASK) { - case CLKCFG_FSB_400: - return 100; - case CLKCFG_FSB_533: - return 133; - case CLKCFG_FSB_667: - return 166; - case CLKCFG_FSB_800: - return 200; - case CLKCFG_FSB_1067: - return 266; - case CLKCFG_FSB_1333: - return 333; - /* these two are just a guess; one of them might be right */ - case CLKCFG_FSB_1600: - case CLKCFG_FSB_1600_ALT: - return 400; - default: - return 133; - } -} - static void intel_dp_init_panel_power_sequencer(struct drm_device *dev, struct intel_dp *intel_dp); @@ -304,7 +277,7 @@ static void pps_lock(struct intel_dp *intel_dp) * See vlv_power_sequencer_reset() why we need * a power domain reference here. */ - power_domain = intel_display_port_power_domain(encoder); + power_domain = intel_display_port_aux_power_domain(encoder); intel_display_power_get(dev_priv, power_domain); mutex_lock(&dev_priv->pps_mutex); @@ -320,7 +293,7 @@ static void pps_unlock(struct intel_dp *intel_dp) mutex_unlock(&dev_priv->pps_mutex); - power_domain = intel_display_port_power_domain(encoder); + power_domain = intel_display_port_aux_power_domain(encoder); intel_display_power_put(dev_priv, power_domain); } @@ -331,7 +304,9 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe = intel_dp->pps_pipe; - bool pll_enabled; + bool pll_enabled, release_cl_override = false; + enum dpio_phy phy = DPIO_PHY(pipe); + enum dpio_channel ch = vlv_pipe_to_channel(pipe); uint32_t DP; if (WARN(I915_READ(intel_dp->output_reg) & DP_PORT_EN, @@ -361,9 +336,13 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) * The DPLL for the pipe must be enabled for this to work. * So enable temporarily it if it's not already enabled. */ - if (!pll_enabled) + if (!pll_enabled) { + release_cl_override = IS_CHERRYVIEW(dev) && + !chv_phy_powergate_ch(dev_priv, phy, ch, true); + vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev) ? &chv_dpll[0].dpll : &vlv_dpll[0].dpll); + } /* * Similar magic as in intel_dp_enable_port(). @@ -380,8 +359,12 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN); POSTING_READ(intel_dp->output_reg); - if (!pll_enabled) + if (!pll_enabled) { vlv_force_pll_off(dev, pipe); + + if (release_cl_override) + chv_phy_powergate_ch(dev_priv, phy, ch, false); + } } static enum pipe @@ -558,21 +541,27 @@ void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv) } } -static u32 _pp_ctrl_reg(struct intel_dp *intel_dp) +static i915_reg_t +_pp_ctrl_reg(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - if (HAS_PCH_SPLIT(dev)) + if (IS_BROXTON(dev)) + return BXT_PP_CONTROL(0); + else if (HAS_PCH_SPLIT(dev)) return PCH_PP_CONTROL; else return VLV_PIPE_PP_CONTROL(vlv_power_sequencer_pipe(intel_dp)); } -static u32 _pp_stat_reg(struct intel_dp *intel_dp) +static i915_reg_t +_pp_stat_reg(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - if (HAS_PCH_SPLIT(dev)) + if (IS_BROXTON(dev)) + return BXT_PP_STATUS(0); + else if (HAS_PCH_SPLIT(dev)) return PCH_PP_STATUS; else return VLV_PIPE_PP_STATUS(vlv_power_sequencer_pipe(intel_dp)); @@ -587,8 +576,6 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, edp_notifier); struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; - u32 pp_div; - u32 pp_ctrl_reg, pp_div_reg; if (!is_edp(intel_dp) || code != SYS_RESTART) return 0; @@ -597,6 +584,8 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, if (IS_VALLEYVIEW(dev)) { enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + i915_reg_t pp_ctrl_reg, pp_div_reg; + u32 pp_div; pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe); pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe); @@ -665,7 +654,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t ch_ctl = intel_dp->aux_ch_ctl_reg; + i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; uint32_t status; bool done; @@ -692,7 +681,7 @@ static uint32_t i9xx_get_aux_clock_divider(struct intel_dp *intel_dp, int index) * The clock divider is based off the hrawclk, and would like to run at * 2MHz. So, take the hrawclk value and divide by 2 and use that */ - return index ? 0 : intel_hrawclk(dev) / 2; + return index ? 0 : DIV_ROUND_CLOSEST(intel_hrawclk(dev), 2); } static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) @@ -705,9 +694,10 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) return 0; if (intel_dig_port->port == PORT_A) { - return DIV_ROUND_UP(dev_priv->display.get_display_clock_speed(dev), 2000); + return DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 2000); + } else { - return DIV_ROUND_UP(intel_pch_rawclk(dev), 2); + return DIV_ROUND_CLOSEST(intel_pch_rawclk(dev), 2); } } @@ -720,8 +710,8 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) if (intel_dig_port->port == PORT_A) { if (index) return 0; - return DIV_ROUND_CLOSEST(dev_priv->display.get_display_clock_speed(dev), 2000); - } else if (dev_priv->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) { + return DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 2000); + } else if (HAS_PCH_LPT_H(dev_priv)) { /* Workaround for non-ULT HSW */ switch (index) { case 0: return 63; @@ -729,7 +719,7 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) default: return 0; } } else { - return index ? 0 : DIV_ROUND_UP(intel_pch_rawclk(dev), 2); + return index ? 0 : DIV_ROUND_CLOSEST(intel_pch_rawclk(dev), 2); } } @@ -762,7 +752,7 @@ static uint32_t i9xx_get_aux_send_ctl(struct intel_dp *intel_dp, else precharge = 5; - if (IS_BROADWELL(dev) && intel_dp->aux_ch_ctl_reg == DPA_AUX_CH_CTL) + if (IS_BROADWELL(dev) && intel_dig_port->port == PORT_A) timeout = DP_AUX_CH_CTL_TIME_OUT_600us; else timeout = DP_AUX_CH_CTL_TIME_OUT_400us; @@ -801,8 +791,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t ch_ctl = intel_dp->aux_ch_ctl_reg; - uint32_t ch_data = ch_ctl + 4; + i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; uint32_t aux_clock_divider; int i, ret, recv_bytes; uint32_t status; @@ -828,8 +817,6 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, intel_dp_check_edp(intel_dp); - intel_aux_display_runtime_get(dev_priv); - /* Try to wait for any previous AUX channel activity */ for (try = 0; try < 3; try++) { status = I915_READ_NOTRACE(ch_ctl); @@ -839,8 +826,15 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, } if (try == 3) { - WARN(1, "dp_aux_ch not started status 0x%08x\n", - I915_READ(ch_ctl)); + static u32 last_status = -1; + const u32 status = I915_READ(ch_ctl); + + if (status != last_status) { + WARN(1, "dp_aux_ch not started status 0x%08x\n", + status); + last_status = status; + } + ret = -EBUSY; goto out; } @@ -861,7 +855,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, for (try = 0; try < 5; try++) { /* Load the send data into the aux channel data registers */ for (i = 0; i < send_bytes; i += 4) - I915_WRITE(ch_data + i, + I915_WRITE(intel_dp->aux_ch_data_reg[i >> 2], intel_dp_pack_aux(send + i, send_bytes - i)); @@ -925,13 +919,12 @@ done: recv_bytes = recv_size; for (i = 0; i < recv_bytes; i += 4) - intel_dp_unpack_aux(I915_READ(ch_data + i), + intel_dp_unpack_aux(I915_READ(intel_dp->aux_ch_data_reg[i >> 2]), recv + i, recv_bytes - i); ret = recv_bytes; out: pm_qos_update_request(&dev_priv->pm_qos, PM_QOS_DEFAULT_VALUE); - intel_aux_display_runtime_put(dev_priv); if (vdd) edp_panel_vdd_off(intel_dp, false); @@ -960,6 +953,7 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) switch (msg->request & ~DP_AUX_I2C_MOT) { case DP_AUX_NATIVE_WRITE: case DP_AUX_I2C_WRITE: + case DP_AUX_I2C_WRITE_STATUS_UPDATE: txsize = msg->size ? HEADER_SIZE + msg->size : BARE_ADDRESS_SIZE; rxsize = 2; /* 0 or 1 data bytes */ @@ -1012,69 +1006,206 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) return ret; } -static void -intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector) +static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv, + enum port port) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - enum port port = intel_dig_port->port; - const char *name = NULL; - int ret; + switch (port) { + case PORT_B: + case PORT_C: + case PORT_D: + return DP_AUX_CH_CTL(port); + default: + MISSING_CASE(port); + return DP_AUX_CH_CTL(PORT_B); + } +} +static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv, + enum port port, int index) +{ + switch (port) { + case PORT_B: + case PORT_C: + case PORT_D: + return DP_AUX_CH_DATA(port, index); + default: + MISSING_CASE(port); + return DP_AUX_CH_DATA(PORT_B, index); + } +} + +static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv, + enum port port) +{ switch (port) { case PORT_A: - intel_dp->aux_ch_ctl_reg = DPA_AUX_CH_CTL; - name = "DPDDC-A"; - break; + return DP_AUX_CH_CTL(port); case PORT_B: - intel_dp->aux_ch_ctl_reg = PCH_DPB_AUX_CH_CTL; - name = "DPDDC-B"; - break; case PORT_C: - intel_dp->aux_ch_ctl_reg = PCH_DPC_AUX_CH_CTL; - name = "DPDDC-C"; - break; case PORT_D: - intel_dp->aux_ch_ctl_reg = PCH_DPD_AUX_CH_CTL; - name = "DPDDC-D"; - break; + return PCH_DP_AUX_CH_CTL(port); default: - BUG(); + MISSING_CASE(port); + return DP_AUX_CH_CTL(PORT_A); } +} - /* - * The AUX_CTL register is usually DP_CTL + 0x10. - * - * On Haswell and Broadwell though: - * - Both port A DDI_BUF_CTL and DDI_AUX_CTL are on the CPU - * - Port B/C/D AUX channels are on the PCH, DDI_BUF_CTL on the CPU - * - * Skylake moves AUX_CTL back next to DDI_BUF_CTL, on the CPU. - */ - if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) - intel_dp->aux_ch_ctl_reg = intel_dp->output_reg + 0x10; +static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv, + enum port port, int index) +{ + switch (port) { + case PORT_A: + return DP_AUX_CH_DATA(port, index); + case PORT_B: + case PORT_C: + case PORT_D: + return PCH_DP_AUX_CH_DATA(port, index); + default: + MISSING_CASE(port); + return DP_AUX_CH_DATA(PORT_A, index); + } +} + +/* + * On SKL we don't have Aux for port E so we rely + * on VBT to set a proper alternate aux channel. + */ +static enum port skl_porte_aux_port(struct drm_i915_private *dev_priv) +{ + const struct ddi_vbt_port_info *info = + &dev_priv->vbt.ddi_port_info[PORT_E]; + + switch (info->alternate_aux_channel) { + case DP_AUX_A: + return PORT_A; + case DP_AUX_B: + return PORT_B; + case DP_AUX_C: + return PORT_C; + case DP_AUX_D: + return PORT_D; + default: + MISSING_CASE(info->alternate_aux_channel); + return PORT_A; + } +} + +static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv, + enum port port) +{ + if (port == PORT_E) + port = skl_porte_aux_port(dev_priv); + + switch (port) { + case PORT_A: + case PORT_B: + case PORT_C: + case PORT_D: + return DP_AUX_CH_CTL(port); + default: + MISSING_CASE(port); + return DP_AUX_CH_CTL(PORT_A); + } +} + +static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv, + enum port port, int index) +{ + if (port == PORT_E) + port = skl_porte_aux_port(dev_priv); + + switch (port) { + case PORT_A: + case PORT_B: + case PORT_C: + case PORT_D: + return DP_AUX_CH_DATA(port, index); + default: + MISSING_CASE(port); + return DP_AUX_CH_DATA(PORT_A, index); + } +} + +static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv, + enum port port) +{ + if (INTEL_INFO(dev_priv)->gen >= 9) + return skl_aux_ctl_reg(dev_priv, port); + else if (HAS_PCH_SPLIT(dev_priv)) + return ilk_aux_ctl_reg(dev_priv, port); + else + return g4x_aux_ctl_reg(dev_priv, port); +} + +static i915_reg_t intel_aux_data_reg(struct drm_i915_private *dev_priv, + enum port port, int index) +{ + if (INTEL_INFO(dev_priv)->gen >= 9) + return skl_aux_data_reg(dev_priv, port, index); + else if (HAS_PCH_SPLIT(dev_priv)) + return ilk_aux_data_reg(dev_priv, port, index); + else + return g4x_aux_data_reg(dev_priv, port, index); +} + +static void intel_aux_reg_init(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); + enum port port = dp_to_dig_port(intel_dp)->port; + int i; + + intel_dp->aux_ch_ctl_reg = intel_aux_ctl_reg(dev_priv, port); + for (i = 0; i < ARRAY_SIZE(intel_dp->aux_ch_data_reg); i++) + intel_dp->aux_ch_data_reg[i] = intel_aux_data_reg(dev_priv, port, i); +} + +static void +intel_dp_aux_fini(struct intel_dp *intel_dp) +{ + drm_dp_aux_unregister(&intel_dp->aux); + kfree(intel_dp->aux.name); +} + +static int +intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector) +{ + struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + enum port port = intel_dig_port->port; + int ret; + + intel_aux_reg_init(intel_dp); + + intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c", port_name(port)); + if (!intel_dp->aux.name) + return -ENOMEM; - intel_dp->aux.name = name; intel_dp->aux.dev = dev->dev; intel_dp->aux.transfer = intel_dp_aux_transfer; - DRM_DEBUG_KMS("registering %s bus for %s\n", name, + DRM_DEBUG_KMS("registering %s bus for %s\n", + intel_dp->aux.name, connector->base.kdev->kobj.name); ret = drm_dp_aux_register(&intel_dp->aux); if (ret < 0) { DRM_ERROR("drm_dp_aux_register() for %s failed (%d)\n", - name, ret); - return; + intel_dp->aux.name, ret); + kfree(intel_dp->aux.name); + return ret; } ret = sysfs_create_link(&connector->base.kdev->kobj, &intel_dp->aux.ddc.dev.kobj, intel_dp->aux.ddc.dev.kobj.name); if (ret < 0) { - DRM_ERROR("sysfs_create_link() for %s failed (%d)\n", name, ret); - drm_dp_aux_unregister(&intel_dp->aux); + DRM_ERROR("sysfs_create_link() for %s failed (%d)\n", + intel_dp->aux.name, ret); + intel_dp_aux_fini(intel_dp); + return ret; } + + return 0; } static void @@ -1089,7 +1220,7 @@ intel_dp_connector_unregister(struct intel_connector *intel_connector) } static void -skl_edp_set_pll_config(struct intel_crtc_state *pipe_config, int link_clock) +skl_edp_set_pll_config(struct intel_crtc_state *pipe_config) { u32 ctrl1; @@ -1101,7 +1232,7 @@ skl_edp_set_pll_config(struct intel_crtc_state *pipe_config, int link_clock) pipe_config->dpll_hw_state.cfgcr2 = 0; ctrl1 = DPLL_CTRL1_OVERRIDE(SKL_DPLL0); - switch (link_clock / 2) { + switch (pipe_config->port_clock / 2) { case 81000: ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0); @@ -1134,20 +1265,20 @@ skl_edp_set_pll_config(struct intel_crtc_state *pipe_config, int link_clock) pipe_config->dpll_hw_state.ctrl1 = ctrl1; } -static void -hsw_dp_set_ddi_pll_sel(struct intel_crtc_state *pipe_config, int link_bw) +void +hsw_dp_set_ddi_pll_sel(struct intel_crtc_state *pipe_config) { memset(&pipe_config->dpll_hw_state, 0, sizeof(pipe_config->dpll_hw_state)); - switch (link_bw) { - case DP_LINK_BW_1_62: + switch (pipe_config->port_clock / 2) { + case 81000: pipe_config->ddi_pll_sel = PORT_CLK_SEL_LCPLL_810; break; - case DP_LINK_BW_2_7: + case 135000: pipe_config->ddi_pll_sel = PORT_CLK_SEL_LCPLL_1350; break; - case DP_LINK_BW_5_4: + case 270000: pipe_config->ddi_pll_sel = PORT_CLK_SEL_LCPLL_2700; break; } @@ -1166,10 +1297,13 @@ intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates) return (intel_dp_max_link_bw(intel_dp) >> 3) + 1; } -static bool intel_dp_source_supports_hbr2(struct drm_device *dev) +bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) { + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = dig_port->base.base.dev; + /* WaDisableHBR2:skl */ - if (IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) + if (IS_SKL_REVID(dev, 0, SKL_REVID_B0)) return false; if ((IS_HASWELL(dev) && !IS_HSW_ULX(dev)) || IS_BROADWELL(dev) || @@ -1180,25 +1314,33 @@ static bool intel_dp_source_supports_hbr2(struct drm_device *dev) } static int -intel_dp_source_rates(struct drm_device *dev, const int **source_rates) +intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates) { - if (IS_SKYLAKE(dev)) { + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = dig_port->base.base.dev; + int size; + + if (IS_BROXTON(dev)) { + *source_rates = bxt_rates; + size = ARRAY_SIZE(bxt_rates); + } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { *source_rates = skl_rates; - return ARRAY_SIZE(skl_rates); + size = ARRAY_SIZE(skl_rates); + } else { + *source_rates = default_rates; + size = ARRAY_SIZE(default_rates); } - *source_rates = default_rates; - /* This depends on the fact that 5.4 is last value in the array */ - if (intel_dp_source_supports_hbr2(dev)) - return (DP_LINK_BW_5_4 >> 3) + 1; - else - return (DP_LINK_BW_2_7 >> 3) + 1; + if (!intel_dp_source_supports_hbr2(intel_dp)) + size--; + + return size; } static void intel_dp_set_clock(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config, int link_bw) + struct intel_crtc_state *pipe_config) { struct drm_device *dev = encoder->base.dev; const struct dp_link_dpll *divisor = NULL; @@ -1220,7 +1362,7 @@ intel_dp_set_clock(struct intel_encoder *encoder, if (divisor && count) { for (i = 0; i < count; i++) { - if (link_bw == divisor[i].link_bw) { + if (pipe_config->port_clock == divisor[i].clock) { pipe_config->dpll = divisor[i].dpll; pipe_config->clock_set = true; break; @@ -1255,12 +1397,11 @@ static int intersect_rates(const int *source_rates, int source_len, static int intel_dp_common_rates(struct intel_dp *intel_dp, int *common_rates) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); const int *source_rates, *sink_rates; int source_len, sink_len; sink_len = intel_dp_sink_rates(intel_dp, &sink_rates); - source_len = intel_dp_source_rates(dev, &source_rates); + source_len = intel_dp_source_rates(intel_dp, &source_rates); return intersect_rates(source_rates, source_len, sink_rates, sink_len, @@ -1285,7 +1426,6 @@ static void snprintf_int_array(char *str, size_t len, static void intel_dp_print_rates(struct intel_dp *intel_dp) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); const int *source_rates, *sink_rates; int source_len, sink_len, common_len; int common_rates[DP_MAX_SUPPORTED_RATES]; @@ -1294,7 +1434,7 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp) if ((drm_debug & DRM_UT_KMS) == 0) return; - source_len = intel_dp_source_rates(dev, &source_rates); + source_len = intel_dp_source_rates(intel_dp, &source_rates); snprintf_int_array(str, sizeof(str), source_rates, source_len); DRM_DEBUG_KMS("source rates: %s\n", str); @@ -1336,6 +1476,19 @@ int intel_dp_rate_select(struct intel_dp *intel_dp, int rate) return rate_to_index(rate, intel_dp->sink_rates); } +void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, + uint8_t *link_bw, uint8_t *rate_select) +{ + if (intel_dp->num_sink_rates) { + *link_bw = 0; + *rate_select = + intel_dp_rate_select(intel_dp, port_clock); + } else { + *link_bw = drm_dp_link_rate_to_bw_code(port_clock); + *rate_select = 0; + } +} + bool intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) @@ -1357,6 +1510,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, int link_avail, link_clock; int common_rates[DP_MAX_SUPPORTED_RATES] = {}; int common_len; + uint8_t link_bw, rate_select; common_len = intel_dp_common_rates(intel_dp, common_rates); @@ -1378,12 +1532,12 @@ intel_dp_compute_config(struct intel_encoder *encoder, if (INTEL_INFO(dev)->gen >= 9) { int ret; - ret = skl_update_scaler_users(intel_crtc, pipe_config, NULL, NULL, 0); + ret = skl_update_scaler_crtc(pipe_config); if (ret) return ret; } - if (!HAS_PCH_SPLIT(dev)) + if (HAS_GMCH_DISPLAY(dev)) intel_gmch_panel_fitting(intel_crtc, pipe_config, intel_connector->panel.fitting_mode); else @@ -1403,7 +1557,10 @@ intel_dp_compute_config(struct intel_encoder *encoder, * bpc in between. */ bpp = pipe_config->pipe_bpp; if (is_edp(intel_dp)) { - if (dev_priv->vbt.edp_bpp && dev_priv->vbt.edp_bpp < bpp) { + + /* Get bpp from vbt only for panels that dont have bpp in edid */ + if (intel_connector->base.display_info.bpc == 0 && + (dev_priv->vbt.edp_bpp && dev_priv->vbt.edp_bpp < bpp)) { DRM_DEBUG_KMS("clamping bpp for eDP panel to BIOS-provided %i\n", dev_priv->vbt.edp_bpp); bpp = dev_priv->vbt.edp_bpp; @@ -1449,32 +1606,23 @@ found: * CEA-861-E - 5.1 Default Encoding Parameters * VESA DisplayPort Ver.1.2a - 5.1.1.1 Video Colorimetry */ - if (bpp != 18 && drm_match_cea_mode(adjusted_mode) > 1) - intel_dp->color_range = DP_COLOR_RANGE_16_235; - else - intel_dp->color_range = 0; - } - - if (intel_dp->color_range) - pipe_config->limited_color_range = true; - - intel_dp->lane_count = lane_count; - - if (intel_dp->num_sink_rates) { - intel_dp->link_bw = 0; - intel_dp->rate_select = - intel_dp_rate_select(intel_dp, common_rates[clock]); + pipe_config->limited_color_range = + bpp != 18 && drm_match_cea_mode(adjusted_mode) > 1; } else { - intel_dp->link_bw = - drm_dp_link_rate_to_bw_code(common_rates[clock]); - intel_dp->rate_select = 0; + pipe_config->limited_color_range = + intel_dp->limited_color_range; } + pipe_config->lane_count = lane_count; + pipe_config->pipe_bpp = bpp; pipe_config->port_clock = common_rates[clock]; - DRM_DEBUG_KMS("DP link bw %02x lane count %d clock %d bpp %d\n", - intel_dp->link_bw, intel_dp->lane_count, + intel_dp_compute_rate(intel_dp, pipe_config->port_clock, + &link_bw, &rate_select); + + DRM_DEBUG_KMS("DP link bw %02x rate select %02x lane count %d clock %d bpp %d\n", + link_bw, rate_select, pipe_config->lane_count, pipe_config->port_clock, bpp); DRM_DEBUG_KMS("DP link bw required %i available %i\n", mode_rate, link_avail); @@ -1493,47 +1641,23 @@ found: &pipe_config->dp_m2_n2); } - if (IS_SKYLAKE(dev) && is_edp(intel_dp)) - skl_edp_set_pll_config(pipe_config, common_rates[clock]); + if ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) && is_edp(intel_dp)) + skl_edp_set_pll_config(pipe_config); else if (IS_BROXTON(dev)) /* handled in ddi */; else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - hsw_dp_set_ddi_pll_sel(pipe_config, intel_dp->link_bw); + hsw_dp_set_ddi_pll_sel(pipe_config); else - intel_dp_set_clock(encoder, pipe_config, intel_dp->link_bw); + intel_dp_set_clock(encoder, pipe_config); return true; } -static void ironlake_set_pll_cpu_edp(struct intel_dp *intel_dp) +void intel_dp_set_link_params(struct intel_dp *intel_dp, + const struct intel_crtc_state *pipe_config) { - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc); - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - u32 dpa_ctl; - - DRM_DEBUG_KMS("eDP PLL enable for clock %d\n", - crtc->config->port_clock); - dpa_ctl = I915_READ(DP_A); - dpa_ctl &= ~DP_PLL_FREQ_MASK; - - if (crtc->config->port_clock == 162000) { - /* For a long time we've carried around a ILK-DevA w/a for the - * 160MHz clock. If we're really unlucky, it's still required. - */ - DRM_DEBUG_KMS("160MHz cpu eDP clock, might need ilk devA w/a\n"); - dpa_ctl |= DP_PLL_FREQ_160MHZ; - intel_dp->DP |= DP_PLL_FREQ_160MHZ; - } else { - dpa_ctl |= DP_PLL_FREQ_270MHZ; - intel_dp->DP |= DP_PLL_FREQ_270MHZ; - } - - I915_WRITE(DP_A, dpa_ctl); - - POSTING_READ(DP_A); - udelay(500); + intel_dp->link_rate = pipe_config->port_clock; + intel_dp->lane_count = pipe_config->lane_count; } static void intel_dp_prepare(struct intel_encoder *encoder) @@ -1543,7 +1667,9 @@ static void intel_dp_prepare(struct intel_encoder *encoder) struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + + intel_dp_set_link_params(intel_dp, crtc->config); /* * There are four kinds of DP registers: @@ -1569,10 +1695,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder) /* Handle DP bits in common between all three register formats */ intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0; - intel_dp->DP |= DP_PORT_WIDTH(intel_dp->lane_count); - - if (crtc->config->has_audio) - intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE; + intel_dp->DP |= DP_PORT_WIDTH(crtc->config->lane_count); /* Split out the IBX/CPU vs CPT settings */ @@ -1599,8 +1722,9 @@ static void intel_dp_prepare(struct intel_encoder *encoder) trans_dp &= ~TRANS_DP_ENH_FRAMING; I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp); } else { - if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev)) - intel_dp->DP |= intel_dp->color_range; + if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) && + crtc->config->limited_color_range) + intel_dp->DP |= DP_COLOR_RANGE_16_235; if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) intel_dp->DP |= DP_SYNC_HS_HIGH; @@ -1633,7 +1757,7 @@ static void wait_panel_status(struct intel_dp *intel_dp, { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; - u32 pp_stat_reg, pp_ctrl_reg; + i915_reg_t pp_stat_reg, pp_ctrl_reg; lockdep_assert_held(&dev_priv->pps_mutex); @@ -1703,8 +1827,10 @@ static u32 ironlake_get_pp_control(struct intel_dp *intel_dp) lockdep_assert_held(&dev_priv->pps_mutex); control = I915_READ(_pp_ctrl_reg(intel_dp)); - control &= ~PANEL_UNLOCK_MASK; - control |= PANEL_UNLOCK_REGS; + if (!IS_BROXTON(dev)) { + control &= ~PANEL_UNLOCK_MASK; + control |= PANEL_UNLOCK_REGS; + } return control; } @@ -1721,7 +1847,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dev->dev_private; enum intel_display_power_domain power_domain; u32 pp; - u32 pp_stat_reg, pp_ctrl_reg; + i915_reg_t pp_stat_reg, pp_ctrl_reg; bool need_to_disable = !intel_dp->want_panel_vdd; lockdep_assert_held(&dev_priv->pps_mutex); @@ -1735,7 +1861,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp) if (edp_have_panel_vdd(intel_dp)) return need_to_disable; - power_domain = intel_display_port_power_domain(intel_encoder); + power_domain = intel_display_port_aux_power_domain(intel_encoder); intel_display_power_get(dev_priv, power_domain); DRM_DEBUG_KMS("Turning eDP port %c VDD on\n", @@ -1797,7 +1923,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) struct intel_encoder *intel_encoder = &intel_dig_port->base; enum intel_display_power_domain power_domain; u32 pp; - u32 pp_stat_reg, pp_ctrl_reg; + i915_reg_t pp_stat_reg, pp_ctrl_reg; lockdep_assert_held(&dev_priv->pps_mutex); @@ -1825,7 +1951,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) if ((pp & POWER_TARGET_ON) == 0) intel_dp->last_power_cycle = jiffies; - power_domain = intel_display_port_power_domain(intel_encoder); + power_domain = intel_display_port_aux_power_domain(intel_encoder); intel_display_power_put(dev_priv, power_domain); } @@ -1884,7 +2010,7 @@ static void edp_panel_on(struct intel_dp *intel_dp) struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; - u32 pp_ctrl_reg; + i915_reg_t pp_ctrl_reg; lockdep_assert_held(&dev_priv->pps_mutex); @@ -1946,7 +2072,7 @@ static void edp_panel_off(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dev->dev_private; enum intel_display_power_domain power_domain; u32 pp; - u32 pp_ctrl_reg; + i915_reg_t pp_ctrl_reg; lockdep_assert_held(&dev_priv->pps_mutex); @@ -1976,7 +2102,7 @@ static void edp_panel_off(struct intel_dp *intel_dp) wait_panel_off(intel_dp); /* We got a reference when we enabled the VDD. */ - power_domain = intel_display_port_power_domain(intel_encoder); + power_domain = intel_display_port_aux_power_domain(intel_encoder); intel_display_power_put(dev_priv, power_domain); } @@ -1997,7 +2123,7 @@ static void _intel_edp_backlight_on(struct intel_dp *intel_dp) struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; - u32 pp_ctrl_reg; + i915_reg_t pp_ctrl_reg; /* * If we enable the backlight right away following a panel power @@ -2038,7 +2164,7 @@ static void _intel_edp_backlight_off(struct intel_dp *intel_dp) struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; - u32 pp_ctrl_reg; + i915_reg_t pp_ctrl_reg; if (!is_edp(intel_dp)) return; @@ -2097,27 +2223,61 @@ static void intel_edp_backlight_power(struct intel_connector *connector, _intel_edp_backlight_off(intel_dp); } +static const char *state_string(bool enabled) +{ + return enabled ? "on" : "off"; +} + +static void assert_dp_port(struct intel_dp *intel_dp, bool state) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + bool cur_state = I915_READ(intel_dp->output_reg) & DP_PORT_EN; + + I915_STATE_WARN(cur_state != state, + "DP port %c state assertion failure (expected %s, current %s)\n", + port_name(dig_port->port), + state_string(state), state_string(cur_state)); +} +#define assert_dp_port_disabled(d) assert_dp_port((d), false) + +static void assert_edp_pll(struct drm_i915_private *dev_priv, bool state) +{ + bool cur_state = I915_READ(DP_A) & DP_PLL_ENABLE; + + I915_STATE_WARN(cur_state != state, + "eDP PLL state assertion failure (expected %s, current %s)\n", + state_string(state), state_string(cur_state)); +} +#define assert_edp_pll_enabled(d) assert_edp_pll((d), true) +#define assert_edp_pll_disabled(d) assert_edp_pll((d), false) + static void ironlake_edp_pll_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_crtc *crtc = intel_dig_port->base.base.crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - u32 dpa_ctl; + struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - assert_pipe_disabled(dev_priv, - to_intel_crtc(crtc)->pipe); + assert_pipe_disabled(dev_priv, crtc->pipe); + assert_dp_port_disabled(intel_dp); + assert_edp_pll_disabled(dev_priv); + + DRM_DEBUG_KMS("enabling eDP PLL for clock %d\n", + crtc->config->port_clock); + + intel_dp->DP &= ~DP_PLL_FREQ_MASK; + + if (crtc->config->port_clock == 162000) + intel_dp->DP |= DP_PLL_FREQ_162MHZ; + else + intel_dp->DP |= DP_PLL_FREQ_270MHZ; + + I915_WRITE(DP_A, intel_dp->DP); + POSTING_READ(DP_A); + udelay(500); - DRM_DEBUG_KMS("\n"); - dpa_ctl = I915_READ(DP_A); - WARN(dpa_ctl & DP_PLL_ENABLE, "dp pll on, should be off\n"); - WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n"); - - /* We don't adjust intel_dp->DP while tearing down the link, to - * facilitate link retraining (e.g. after hotplug). Hence clear all - * enable bits here to ensure that we don't enable too much. */ - intel_dp->DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE); intel_dp->DP |= DP_PLL_ENABLE; + I915_WRITE(DP_A, intel_dp->DP); POSTING_READ(DP_A); udelay(200); @@ -2126,24 +2286,18 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp) static void ironlake_edp_pll_off(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_crtc *crtc = intel_dig_port->base.base.crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - u32 dpa_ctl; + struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - assert_pipe_disabled(dev_priv, - to_intel_crtc(crtc)->pipe); + assert_pipe_disabled(dev_priv, crtc->pipe); + assert_dp_port_disabled(intel_dp); + assert_edp_pll_enabled(dev_priv); - dpa_ctl = I915_READ(DP_A); - WARN((dpa_ctl & DP_PLL_ENABLE) == 0, - "dp pll off, should be on\n"); - WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n"); + DRM_DEBUG_KMS("disabling eDP PLL\n"); - /* We can't rely on the value tracked for the DP register in - * intel_dp->DP because link_down must not change that (otherwise link - * re-training will fail. */ - dpa_ctl &= ~DP_PLL_ENABLE; - I915_WRITE(DP_A, dpa_ctl); + intel_dp->DP &= ~DP_PLL_ENABLE; + + I915_WRITE(DP_A, intel_dp->DP); POSTING_READ(DP_A); udelay(200); } @@ -2212,7 +2366,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder, } DRM_DEBUG_KMS("No pipe for dp port 0x%x found\n", - intel_dp->output_reg); + i915_mmio_reg_offset(intel_dp->output_reg)); } else if (IS_CHERRYVIEW(dev)) { *pipe = DP_PORT_TO_PIPE_CHV(tmp); } else { @@ -2238,13 +2392,14 @@ static void intel_dp_get_config(struct intel_encoder *encoder, pipe_config->has_audio = tmp & DP_AUDIO_OUTPUT_ENABLE && port != PORT_A; if (HAS_PCH_CPT(dev) && port != PORT_A) { - tmp = I915_READ(TRANS_DP_CTL(crtc->pipe)); - if (tmp & TRANS_DP_HSYNC_ACTIVE_HIGH) + u32 trans_dp = I915_READ(TRANS_DP_CTL(crtc->pipe)); + + if (trans_dp & TRANS_DP_HSYNC_ACTIVE_HIGH) flags |= DRM_MODE_FLAG_PHSYNC; else flags |= DRM_MODE_FLAG_NHSYNC; - if (tmp & TRANS_DP_VSYNC_ACTIVE_HIGH) + if (trans_dp & TRANS_DP_VSYNC_ACTIVE_HIGH) flags |= DRM_MODE_FLAG_PVSYNC; else flags |= DRM_MODE_FLAG_NVSYNC; @@ -2268,10 +2423,13 @@ static void intel_dp_get_config(struct intel_encoder *encoder, pipe_config->has_dp_encoder = true; + pipe_config->lane_count = + ((tmp & DP_PORT_WIDTH_MASK) >> DP_PORT_WIDTH_SHIFT) + 1; + intel_dp_get_m_n(crtc, pipe_config); if (port == PORT_A) { - if ((I915_READ(DP_A) & DP_PLL_FREQ_MASK) == DP_PLL_FREQ_160MHZ) + if ((I915_READ(DP_A) & DP_PLL_FREQ_MASK) == DP_PLL_FREQ_162MHZ) pipe_config->port_clock = 162000; else pipe_config->port_clock = 270000; @@ -2336,6 +2494,8 @@ static void ilk_post_disable_dp(struct intel_encoder *encoder) enum port port = dp_to_dig_port(intel_dp)->port; intel_dp_link_down(intel_dp); + + /* Only ilk+ has port A */ if (port == PORT_A) ironlake_edp_pll_off(intel_dp); } @@ -2347,38 +2507,62 @@ static void vlv_post_disable_dp(struct intel_encoder *encoder) intel_dp_link_down(intel_dp); } -static void chv_post_disable_dp(struct intel_encoder *encoder) +static void chv_data_lane_soft_reset(struct intel_encoder *encoder, + bool reset) { - struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel ch = vlv_dport_to_channel(dport); - enum pipe pipe = intel_crtc->pipe; - u32 val; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + enum pipe pipe = crtc->pipe; + uint32_t val; - intel_dp_link_down(intel_dp); + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + if (reset) + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + else + val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - mutex_lock(&dev_priv->sb_lock); + if (crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + if (reset) + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + else + val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + } - /* Propagate soft reset to data lane reset */ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); val |= CHV_PCS_REQ_SOFTRESET_EN; + if (reset) + val &= ~DPIO_PCS_CLK_SOFT_RESET; + else + val |= DPIO_PCS_CLK_SOFT_RESET; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + if (crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + if (reset) + val &= ~DPIO_PCS_CLK_SOFT_RESET; + else + val |= DPIO_PCS_CLK_SOFT_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + } +} - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); +static void chv_post_disable_dp(struct intel_encoder *encoder) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + intel_dp_link_down(intel_dp); + + mutex_lock(&dev_priv->sb_lock); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + /* Assert data lane reset */ + chv_data_lane_soft_reset(encoder, true); mutex_unlock(&dev_priv->sb_lock); } @@ -2471,6 +2655,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc = + to_intel_crtc(dp_to_dig_port(intel_dp)->base.base.crtc); /* enable with pattern 1 (as per spec) */ _intel_dp_set_link_train(intel_dp, &intel_dp->DP, @@ -2486,6 +2672,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp) * fail when the power sequencer is freshly used for this port. */ intel_dp->DP |= DP_PORT_EN; + if (crtc->config->has_audio) + intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE; I915_WRITE(intel_dp->output_reg, intel_dp->DP); POSTING_READ(intel_dp->output_reg); @@ -2498,7 +2686,8 @@ static void intel_enable_dp(struct intel_encoder *encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); uint32_t dp_reg = I915_READ(intel_dp->output_reg); - unsigned int lane_mask = 0x0; + enum port port = dp_to_dig_port(intel_dp)->port; + enum pipe pipe = crtc->pipe; if (WARN_ON(dp_reg & DP_PORT_EN)) return; @@ -2508,26 +2697,54 @@ static void intel_enable_dp(struct intel_encoder *encoder) if (IS_VALLEYVIEW(dev)) vlv_init_panel_power_sequencer(intel_dp); + /* + * We get an occasional spurious underrun between the port + * enable and vdd enable, when enabling port A eDP. + * + * FIXME: Not sure if this applies to (PCH) port D eDP as well + */ + if (port == PORT_A) + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); + intel_dp_enable_port(intel_dp); + if (port == PORT_A && IS_GEN5(dev_priv)) { + /* + * Underrun reporting for the other pipe was disabled in + * g4x_pre_enable_dp(). The eDP PLL and port have now been + * enabled, so it's now safe to re-enable underrun reporting. + */ + intel_wait_for_vblank_if_active(dev_priv->dev, !pipe); + intel_set_cpu_fifo_underrun_reporting(dev_priv, !pipe, true); + intel_set_pch_fifo_underrun_reporting(dev_priv, !pipe, true); + } + edp_panel_vdd_on(intel_dp); edp_panel_on(intel_dp); edp_panel_vdd_off(intel_dp, true); + if (port == PORT_A) + intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); + pps_unlock(intel_dp); - if (IS_VALLEYVIEW(dev)) + if (IS_VALLEYVIEW(dev)) { + unsigned int lane_mask = 0x0; + + if (IS_CHERRYVIEW(dev)) + lane_mask = intel_dp_unused_lane_mask(crtc->config->lane_count); + vlv_wait_port_ready(dev_priv, dp_to_dig_port(intel_dp), lane_mask); + } intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); intel_dp_start_link_train(intel_dp); - intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); if (crtc->config->has_audio) { DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n", - pipe_name(crtc->pipe)); + pipe_name(pipe)); intel_audio_codec_enable(encoder); } } @@ -2550,16 +2767,29 @@ static void vlv_enable_dp(struct intel_encoder *encoder) static void g4x_pre_enable_dp(struct intel_encoder *encoder) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); + enum port port = dp_to_dig_port(intel_dp)->port; + enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe; intel_dp_prepare(encoder); + if (port == PORT_A && IS_GEN5(dev_priv)) { + /* + * We get FIFO underruns on the other pipe when + * enabling the CPU eDP PLL, and when enabling CPU + * eDP port. We could potentially avoid the PLL + * underrun with a vblank wait just prior to enabling + * the PLL, but that doesn't appear to help the port + * enable case. Just sweep it all under the rug. + */ + intel_set_cpu_fifo_underrun_reporting(dev_priv, !pipe, false); + intel_set_pch_fifo_underrun_reporting(dev_priv, !pipe, false); + } + /* Only ilk+ has port A */ - if (dport->port == PORT_A) { - ironlake_set_pll_cpu_edp(intel_dp); + if (port == PORT_A) ironlake_edp_pll_on(intel_dp); - } } static void vlv_detach_power_sequencer(struct intel_dp *intel_dp) @@ -2567,7 +2797,7 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = intel_dig_port->base.base.dev->dev_private; enum pipe pipe = intel_dp->pps_pipe; - int pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe); + i915_reg_t pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe); edp_panel_vdd_off_sync(intel_dp); @@ -2616,7 +2846,7 @@ static void vlv_steal_power_sequencer(struct drm_device *dev, DRM_DEBUG_KMS("stealing pipe %c power sequencer from port %c\n", pipe_name(pipe), port_name(port)); - WARN(encoder->connectors_active, + WARN(encoder->base.crtc, "stealing pipe %c power sequencer from active eDP port %c\n", pipe_name(pipe), port_name(port)); @@ -2745,31 +2975,19 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder) val &= ~DPIO_LANEDESKEW_STRAP_OVRD; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); - val &= ~DPIO_LANEDESKEW_STRAP_OVRD; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); - - /* Deassert soft data lane reset*/ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); - val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); - val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); + val &= ~DPIO_LANEDESKEW_STRAP_OVRD; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); + } /* Program Tx lane latency optimal setting*/ - for (i = 0; i < 4; i++) { + for (i = 0; i < intel_crtc->config->lane_count; i++) { /* Set the upar bit */ - data = (i == 1) ? 0x0 : 0x1; + if (intel_crtc->config->lane_count == 1) + data = 0x0; + else + data = (i == 1) ? 0x0 : 0x1; vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), data << DPIO_UPAR_SHIFT); } @@ -2790,9 +3008,11 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder) val |= DPIO_TX2_STAGGER_MASK(0x1f); vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); - val |= DPIO_TX2_STAGGER_MASK(0x1f); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); + val |= DPIO_TX2_STAGGER_MASK(0x1f); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); + } vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch), DPIO_LANESTAGGER_STRAP(stagger) | @@ -2801,16 +3021,27 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder) DPIO_TX1_STAGGER_MULT(6) | DPIO_TX2_STAGGER_MULT(0)); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), - DPIO_LANESTAGGER_STRAP(stagger) | - DPIO_LANESTAGGER_STRAP_OVRD | - DPIO_TX1_STAGGER_MASK(0x1f) | - DPIO_TX1_STAGGER_MULT(7) | - DPIO_TX2_STAGGER_MULT(5)); + if (intel_crtc->config->lane_count > 2) { + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), + DPIO_LANESTAGGER_STRAP(stagger) | + DPIO_LANESTAGGER_STRAP_OVRD | + DPIO_TX1_STAGGER_MASK(0x1f) | + DPIO_TX1_STAGGER_MULT(7) | + DPIO_TX2_STAGGER_MULT(5)); + } + + /* Deassert data lane reset */ + chv_data_lane_soft_reset(encoder, false); mutex_unlock(&dev_priv->sb_lock); intel_enable_dp(encoder); + + /* Second common lane will stay alive on its own now */ + if (dport->release_cl2_override) { + chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); + dport->release_cl2_override = false; + } } static void chv_dp_pre_pll_enable(struct intel_encoder *encoder) @@ -2822,12 +3053,27 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder) to_intel_crtc(encoder->base.crtc); enum dpio_channel ch = vlv_dport_to_channel(dport); enum pipe pipe = intel_crtc->pipe; + unsigned int lane_mask = + intel_dp_unused_lane_mask(intel_crtc->config->lane_count); u32 val; intel_dp_prepare(encoder); + /* + * Must trick the second common lane into life. + * Otherwise we can't even access the PLL. + */ + if (ch == DPIO_CH0 && pipe == PIPE_B) + dport->release_cl2_override = + !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); + + chv_phy_powergate_lanes(encoder, true, lane_mask); + mutex_lock(&dev_priv->sb_lock); + /* Assert data lane reset */ + chv_data_lane_soft_reset(encoder, true); + /* program left/right clock distribution */ if (pipe != PIPE_B) { val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); @@ -2856,13 +3102,15 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder) val |= CHV_PCS_USEDCLKCHANNEL; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); - val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; - if (pipe != PIPE_B) - val &= ~CHV_PCS_USEDCLKCHANNEL; - else - val |= CHV_PCS_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); + val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; + if (pipe != PIPE_B) + val &= ~CHV_PCS_USEDCLKCHANNEL; + else + val |= CHV_PCS_USEDCLKCHANNEL; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); + } /* * This a a bit weird since generally CL @@ -2879,6 +3127,39 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder) mutex_unlock(&dev_priv->sb_lock); } +static void chv_dp_post_pll_disable(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe; + u32 val; + + mutex_lock(&dev_priv->sb_lock); + + /* disable left/right clock distribution */ + if (pipe != PIPE_B) { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); + val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); + } else { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); + val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); + } + + mutex_unlock(&dev_priv->sb_lock); + + /* + * Leave the power down bit cleared for at least one + * lane so that chv_powergate_phy_ch() will power + * on something when the channel is otherwise unused. + * When the port is off and the override is removed + * the lanes power down anyway, so otherwise it doesn't + * really matter what the state of power down bits is + * after this. + */ + chv_phy_powergate_lanes(encoder, false, 0x0); +} + /* * Native read with retry for link status and receiver capability reads for * cases where the sink may still be asleep. @@ -2914,7 +3195,7 @@ intel_dp_dpcd_read_wake(struct drm_dp_aux *aux, unsigned int offset, * Fetch AUX CH registers 0x202 - 0x207 which contain * link status information */ -static bool +bool intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]) { return intel_dp_dpcd_read_wake(&intel_dp->aux, @@ -2924,7 +3205,7 @@ intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_ } /* These are source-specific values. */ -static uint8_t +uint8_t intel_dp_voltage_max(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); @@ -2947,7 +3228,7 @@ intel_dp_voltage_max(struct intel_dp *intel_dp) return DP_TRAIN_VOLTAGE_SWING_LEVEL_2; } -static uint8_t +uint8_t intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing) { struct drm_device *dev = intel_dp_to_dev(intel_dp); @@ -3115,6 +3396,12 @@ static uint32_t vlv_signal_levels(struct intel_dp *intel_dp) return 0; } +static bool chv_need_uniq_trans_scale(uint8_t train_set) +{ + return (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) == DP_TRAIN_PRE_EMPH_LEVEL_0 && + (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) == DP_TRAIN_VOLTAGE_SWING_LEVEL_3; +} + static uint32_t chv_signal_levels(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); @@ -3206,24 +3493,28 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp) val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); - val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); - val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); - val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); + val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + } val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch)); val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); - val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); - val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); + val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); + val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); + } /* Program swing deemph */ - for (i = 0; i < 4; i++) { + for (i = 0; i < intel_crtc->config->lane_count; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); val &= ~DPIO_SWING_DEEMPH9P5_MASK; val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT; @@ -3231,43 +3522,36 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp) } /* Program swing margin */ - for (i = 0; i < 4; i++) { + for (i = 0; i < intel_crtc->config->lane_count; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); + val &= ~DPIO_SWING_MARGIN000_MASK; val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT; + + /* + * Supposedly this value shouldn't matter when unique transition + * scale is disabled, but in fact it does matter. Let's just + * always program the same value and hope it's OK. + */ + val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); + val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); } - /* Disable unique transition scale */ - for (i = 0; i < 4; i++) { + /* + * The document said it needs to set bit 27 for ch0 and bit 26 + * for ch1. Might be a typo in the doc. + * For now, for this unique transition scale selection, set bit + * 27 for ch0 and ch1. + */ + for (i = 0; i < intel_crtc->config->lane_count; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); - val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); - } - - if (((train_set & DP_TRAIN_PRE_EMPHASIS_MASK) - == DP_TRAIN_PRE_EMPH_LEVEL_0) && - ((train_set & DP_TRAIN_VOLTAGE_SWING_MASK) - == DP_TRAIN_VOLTAGE_SWING_LEVEL_3)) { - - /* - * The document said it needs to set bit 27 for ch0 and bit 26 - * for ch1. Might be a typo in the doc. - * For now, for this unique transition scale selection, set bit - * 27 for ch0 and ch1. - */ - for (i = 0; i < 4; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); + if (chv_need_uniq_trans_scale(train_set)) val |= DPIO_TX_UNIQ_TRANS_SCALE_EN; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); - } - - for (i = 0; i < 4; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); - val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); - val |= (0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT); - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); - } + else + val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); } /* Start swing calculation */ @@ -3275,52 +3559,17 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp) val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); - val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); - - /* LRC Bypass */ - val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30); - val |= DPIO_LRC_BYPASS; - vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, val); + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + } mutex_unlock(&dev_priv->sb_lock); return 0; } -static void -intel_get_adjust_train(struct intel_dp *intel_dp, - const uint8_t link_status[DP_LINK_STATUS_SIZE]) -{ - uint8_t v = 0; - uint8_t p = 0; - int lane; - uint8_t voltage_max; - uint8_t preemph_max; - - for (lane = 0; lane < intel_dp->lane_count; lane++) { - uint8_t this_v = drm_dp_get_adjust_request_voltage(link_status, lane); - uint8_t this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); - - if (this_v > v) - v = this_v; - if (this_p > p) - p = this_p; - } - - voltage_max = intel_dp_voltage_max(intel_dp); - if (v >= voltage_max) - v = voltage_max | DP_TRAIN_MAX_SWING_REACHED; - - preemph_max = intel_dp_pre_emphasis_max(intel_dp, v); - if (p >= preemph_max) - p = preemph_max | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED; - - for (lane = 0; lane < 4; lane++) - intel_dp->train_set[lane] = v | p; -} - static uint32_t gen4_signal_levels(uint8_t train_set) { @@ -3418,115 +3667,27 @@ gen7_edp_signal_levels(uint8_t train_set) } } -/* Gen7.5's (HSW) DP voltage swing and pre-emphasis control */ -static uint32_t -hsw_signal_levels(uint8_t train_set) -{ - int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | - DP_TRAIN_PRE_EMPHASIS_MASK); - switch (signal_levels) { - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: - return DDI_BUF_TRANS_SELECT(0); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1: - return DDI_BUF_TRANS_SELECT(1); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2: - return DDI_BUF_TRANS_SELECT(2); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_3: - return DDI_BUF_TRANS_SELECT(3); - - case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0: - return DDI_BUF_TRANS_SELECT(4); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1: - return DDI_BUF_TRANS_SELECT(5); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2: - return DDI_BUF_TRANS_SELECT(6); - - case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0: - return DDI_BUF_TRANS_SELECT(7); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1: - return DDI_BUF_TRANS_SELECT(8); - - case DP_TRAIN_VOLTAGE_SWING_LEVEL_3 | DP_TRAIN_PRE_EMPH_LEVEL_0: - return DDI_BUF_TRANS_SELECT(9); - default: - DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" - "0x%x\n", signal_levels); - return DDI_BUF_TRANS_SELECT(0); - } -} - -static void bxt_signal_levels(struct intel_dp *intel_dp) -{ - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - enum port port = dport->port; - struct drm_device *dev = dport->base.base.dev; - struct intel_encoder *encoder = &dport->base; - uint8_t train_set = intel_dp->train_set[0]; - uint32_t level = 0; - - int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | - DP_TRAIN_PRE_EMPHASIS_MASK); - switch (signal_levels) { - default: - DRM_DEBUG_KMS("Unsupported voltage swing/pre-emph level\n"); - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: - level = 0; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1: - level = 1; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2: - level = 2; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_3: - level = 3; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0: - level = 4; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1: - level = 5; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2: - level = 6; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0: - level = 7; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1: - level = 8; - break; - case DP_TRAIN_VOLTAGE_SWING_LEVEL_3 | DP_TRAIN_PRE_EMPH_LEVEL_0: - level = 9; - break; - } - - bxt_ddi_vswing_sequence(dev, level, port, encoder->type); -} - -/* Properly updates "DP" with the correct signal levels. */ -static void -intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP) +void +intel_dp_set_signal_levels(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); enum port port = intel_dig_port->port; struct drm_device *dev = intel_dig_port->base.base.dev; - uint32_t signal_levels, mask; + struct drm_i915_private *dev_priv = to_i915(dev); + uint32_t signal_levels, mask = 0; uint8_t train_set = intel_dp->train_set[0]; - if (IS_BROXTON(dev)) { - signal_levels = 0; - bxt_signal_levels(intel_dp); - mask = 0; - } else if (HAS_DDI(dev)) { - signal_levels = hsw_signal_levels(train_set); - mask = DDI_BUF_EMP_MASK; + if (HAS_DDI(dev)) { + signal_levels = ddi_signal_levels(intel_dp); + + if (IS_BROXTON(dev)) + signal_levels = 0; + else + mask = DDI_BUF_EMP_MASK; } else if (IS_CHERRYVIEW(dev)) { signal_levels = chv_signal_levels(intel_dp); - mask = 0; } else if (IS_VALLEYVIEW(dev)) { signal_levels = vlv_signal_levels(intel_dp); - mask = 0; } else if (IS_GEN7(dev) && port == PORT_A) { signal_levels = gen7_edp_signal_levels(train_set); mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB; @@ -3547,74 +3708,27 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP) (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) >> DP_TRAIN_PRE_EMPHASIS_SHIFT); - *DP = (*DP & ~mask) | signal_levels; -} - -static bool -intel_dp_set_link_train(struct intel_dp *intel_dp, - uint32_t *DP, - uint8_t dp_train_pat) -{ - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - uint8_t buf[sizeof(intel_dp->train_set) + 1]; - int ret, len; - - _intel_dp_set_link_train(intel_dp, DP, dp_train_pat); + intel_dp->DP = (intel_dp->DP & ~mask) | signal_levels; - I915_WRITE(intel_dp->output_reg, *DP); + I915_WRITE(intel_dp->output_reg, intel_dp->DP); POSTING_READ(intel_dp->output_reg); - - buf[0] = dp_train_pat; - if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) == - DP_TRAINING_PATTERN_DISABLE) { - /* don't write DP_TRAINING_LANEx_SET on disable */ - len = 1; - } else { - /* DP_TRAINING_LANEx_SET follow DP_TRAINING_PATTERN_SET */ - memcpy(buf + 1, intel_dp->train_set, intel_dp->lane_count); - len = intel_dp->lane_count + 1; - } - - ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET, - buf, len); - - return ret == len; } -static bool -intel_dp_reset_link_train(struct intel_dp *intel_dp, uint32_t *DP, - uint8_t dp_train_pat) -{ - if (!intel_dp->train_set_valid) - memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); - intel_dp_set_signal_levels(intel_dp, DP); - return intel_dp_set_link_train(intel_dp, DP, dp_train_pat); -} - -static bool -intel_dp_update_link_train(struct intel_dp *intel_dp, uint32_t *DP, - const uint8_t link_status[DP_LINK_STATUS_SIZE]) +void +intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, + uint8_t dp_train_pat) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; + struct drm_i915_private *dev_priv = + to_i915(intel_dig_port->base.base.dev); - intel_get_adjust_train(intel_dp, link_status); - intel_dp_set_signal_levels(intel_dp, DP); + _intel_dp_set_link_train(intel_dp, &intel_dp->DP, dp_train_pat); - I915_WRITE(intel_dp->output_reg, *DP); + I915_WRITE(intel_dp->output_reg, intel_dp->DP); POSTING_READ(intel_dp->output_reg); - - ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET, - intel_dp->train_set, intel_dp->lane_count); - - return ret == intel_dp->lane_count; } -static void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) +void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; @@ -3645,206 +3759,6 @@ static void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) DRM_ERROR("Timed out waiting for DP idle patterns\n"); } -/* Enable corresponding port and start training pattern 1 */ -void -intel_dp_start_link_train(struct intel_dp *intel_dp) -{ - struct drm_encoder *encoder = &dp_to_dig_port(intel_dp)->base.base; - struct drm_device *dev = encoder->dev; - int i; - uint8_t voltage; - int voltage_tries, loop_tries; - uint32_t DP = intel_dp->DP; - uint8_t link_config[2]; - - if (HAS_DDI(dev)) - intel_ddi_prepare_link_retrain(encoder); - - /* Write the link configuration data */ - link_config[0] = intel_dp->link_bw; - link_config[1] = intel_dp->lane_count; - if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) - link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN; - drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2); - if (intel_dp->num_sink_rates) - drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET, - &intel_dp->rate_select, 1); - - link_config[0] = 0; - link_config[1] = DP_SET_ANSI_8B10B; - drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2); - - DP |= DP_PORT_EN; - - /* clock recovery */ - if (!intel_dp_reset_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_1 | - DP_LINK_SCRAMBLING_DISABLE)) { - DRM_ERROR("failed to enable link training\n"); - return; - } - - voltage = 0xff; - voltage_tries = 0; - loop_tries = 0; - for (;;) { - uint8_t link_status[DP_LINK_STATUS_SIZE]; - - drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd); - if (!intel_dp_get_link_status(intel_dp, link_status)) { - DRM_ERROR("failed to get link status\n"); - break; - } - - if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { - DRM_DEBUG_KMS("clock recovery OK\n"); - break; - } - - /* - * if we used previously trained voltage and pre-emphasis values - * and we don't get clock recovery, reset link training values - */ - if (intel_dp->train_set_valid) { - DRM_DEBUG_KMS("clock recovery not ok, reset"); - /* clear the flag as we are not reusing train set */ - intel_dp->train_set_valid = false; - if (!intel_dp_reset_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_1 | - DP_LINK_SCRAMBLING_DISABLE)) { - DRM_ERROR("failed to enable link training\n"); - return; - } - continue; - } - - /* Check to see if we've tried the max voltage */ - for (i = 0; i < intel_dp->lane_count; i++) - if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0) - break; - if (i == intel_dp->lane_count) { - ++loop_tries; - if (loop_tries == 5) { - DRM_ERROR("too many full retries, give up\n"); - break; - } - intel_dp_reset_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_1 | - DP_LINK_SCRAMBLING_DISABLE); - voltage_tries = 0; - continue; - } - - /* Check to see if we've tried the same voltage 5 times */ - if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) { - ++voltage_tries; - if (voltage_tries == 5) { - DRM_ERROR("too many voltage retries, give up\n"); - break; - } - } else - voltage_tries = 0; - voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; - - /* Update training set as requested by target */ - if (!intel_dp_update_link_train(intel_dp, &DP, link_status)) { - DRM_ERROR("failed to update link training\n"); - break; - } - } - - intel_dp->DP = DP; -} - -void -intel_dp_complete_link_train(struct intel_dp *intel_dp) -{ - bool channel_eq = false; - int tries, cr_tries; - uint32_t DP = intel_dp->DP; - uint32_t training_pattern = DP_TRAINING_PATTERN_2; - - /* Training Pattern 3 for HBR2 ot 1.2 devices that support it*/ - if (intel_dp->link_bw == DP_LINK_BW_5_4 || intel_dp->use_tps3) - training_pattern = DP_TRAINING_PATTERN_3; - - /* channel equalization */ - if (!intel_dp_set_link_train(intel_dp, &DP, - training_pattern | - DP_LINK_SCRAMBLING_DISABLE)) { - DRM_ERROR("failed to start channel equalization\n"); - return; - } - - tries = 0; - cr_tries = 0; - channel_eq = false; - for (;;) { - uint8_t link_status[DP_LINK_STATUS_SIZE]; - - if (cr_tries > 5) { - DRM_ERROR("failed to train DP, aborting\n"); - break; - } - - drm_dp_link_train_channel_eq_delay(intel_dp->dpcd); - if (!intel_dp_get_link_status(intel_dp, link_status)) { - DRM_ERROR("failed to get link status\n"); - break; - } - - /* Make sure clock is still ok */ - if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { - intel_dp->train_set_valid = false; - intel_dp_start_link_train(intel_dp); - intel_dp_set_link_train(intel_dp, &DP, - training_pattern | - DP_LINK_SCRAMBLING_DISABLE); - cr_tries++; - continue; - } - - if (drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) { - channel_eq = true; - break; - } - - /* Try 5 times, then try clock recovery if that fails */ - if (tries > 5) { - intel_dp->train_set_valid = false; - intel_dp_start_link_train(intel_dp); - intel_dp_set_link_train(intel_dp, &DP, - training_pattern | - DP_LINK_SCRAMBLING_DISABLE); - tries = 0; - cr_tries++; - continue; - } - - /* Update training set as requested by target */ - if (!intel_dp_update_link_train(intel_dp, &DP, link_status)) { - DRM_ERROR("failed to update link training\n"); - break; - } - ++tries; - } - - intel_dp_set_idle_link_train(intel_dp); - - intel_dp->DP = DP; - - if (channel_eq) { - intel_dp->train_set_valid = true; - DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n"); - } -} - -void intel_dp_stop_link_train(struct intel_dp *intel_dp) -{ - intel_dp_set_link_train(intel_dp, &intel_dp->DP, - DP_TRAINING_PATTERN_DISABLE); -} - static void intel_dp_link_down(struct intel_dp *intel_dp) { @@ -3887,6 +3801,13 @@ intel_dp_link_down(struct intel_dp *intel_dp) * matching HDMI port to be enabled on transcoder A. */ if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B && port != PORT_A) { + /* + * We get CPU/PCH FIFO underruns on the other pipe when + * doing the workaround. Sweep them under the rug. + */ + intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false); + intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); + /* always enable with pattern 1 (as per spec) */ DP &= ~(DP_PIPEB_SELECT | DP_LINK_TRAIN_MASK); DP |= DP_PORT_EN | DP_LINK_TRAIN_PAT_1; @@ -3896,9 +3817,15 @@ intel_dp_link_down(struct intel_dp *intel_dp) DP &= ~DP_PORT_EN; I915_WRITE(intel_dp->output_reg, DP); POSTING_READ(intel_dp->output_reg); + + intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A); + intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); + intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } msleep(intel_dp->panel_power_down_delay); + + intel_dp->DP = DP; } static bool @@ -3945,19 +3872,9 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) } } - /* Training Pattern 3 support, Intel platforms that support HBR2 alone - * have support for TP3 hence that check is used along with dpcd check - * to ensure TP3 can be enabled. - * SKL < B0: due it's WaDisableHBR2 is the only exception where TP3 is - * supported but still not enabled. - */ - if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 && - intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED && - intel_dp_source_supports_hbr2(dev)) { - intel_dp->use_tps3 = true; - DRM_DEBUG_KMS("Displayport TPS3 supported\n"); - } else - intel_dp->use_tps3 = false; + DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n", + yesno(intel_dp_source_supports_hbr2(intel_dp)), + yesno(drm_dp_tps3_supported(intel_dp->dpcd))); /* Intermediate frequency support */ if (is_edp(intel_dp) && @@ -4043,78 +3960,123 @@ intel_dp_probe_mst(struct intel_dp *intel_dp) return intel_dp->is_mst; } -int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) +static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) { - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - struct intel_crtc *intel_crtc = - to_intel_crtc(intel_dig_port->base.base.crtc); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = dig_port->base.base.dev; + struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; - int test_crc_count; - int attempts = 6; int ret = 0; + int count = 0; + int attempts = 10; - hsw_disable_ips(intel_crtc); - - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { + if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) { + DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n"); ret = -EIO; goto out; } - if (!(buf & DP_TEST_CRC_SUPPORTED)) { - ret = -ENOTTY; + if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, + buf & ~DP_TEST_SINK_START) < 0) { + DRM_DEBUG_KMS("Sink CRC couldn't be stopped properly\n"); + ret = -EIO; goto out; } - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) { - ret = -EIO; - goto out; + do { + intel_wait_for_vblank(dev, intel_crtc->pipe); + + if (drm_dp_dpcd_readb(&intel_dp->aux, + DP_TEST_SINK_MISC, &buf) < 0) { + ret = -EIO; + goto out; + } + count = buf & DP_TEST_COUNT_MASK; + } while (--attempts && count); + + if (attempts == 0) { + DRM_ERROR("TIMEOUT: Sink CRC counter is not zeroed\n"); + ret = -ETIMEDOUT; } - if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, - buf | DP_TEST_SINK_START) < 0) { - ret = -EIO; - goto out; + out: + hsw_enable_ips(intel_crtc); + return ret; +} + +static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = dig_port->base.base.dev; + struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); + u8 buf; + int ret; + + if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) + return -EIO; + + if (!(buf & DP_TEST_CRC_SUPPORTED)) + return -ENOTTY; + + if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) + return -EIO; + + if (buf & DP_TEST_SINK_START) { + ret = intel_dp_sink_crc_stop(intel_dp); + if (ret) + return ret; } - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { - ret = -EIO; - goto out; + hsw_disable_ips(intel_crtc); + + if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, + buf | DP_TEST_SINK_START) < 0) { + hsw_enable_ips(intel_crtc); + return -EIO; } - test_crc_count = buf & DP_TEST_COUNT_MASK; + intel_wait_for_vblank(dev, intel_crtc->pipe); + return 0; +} + +int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = dig_port->base.base.dev; + struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); + u8 buf; + int count, ret; + int attempts = 6; + + ret = intel_dp_sink_crc_start(intel_dp); + if (ret) + return ret; do { + intel_wait_for_vblank(dev, intel_crtc->pipe); + if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { ret = -EIO; - goto out; + goto stop; } - intel_wait_for_vblank(dev, intel_crtc->pipe); - } while (--attempts && (buf & DP_TEST_COUNT_MASK) == test_crc_count); + count = buf & DP_TEST_COUNT_MASK; + + } while (--attempts && count == 0); if (attempts == 0) { - DRM_DEBUG_KMS("Panel is unable to calculate CRC after 6 vblanks\n"); + DRM_ERROR("Panel is unable to calculate any CRC after 6 vblanks\n"); ret = -ETIMEDOUT; - goto out; + goto stop; } if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) { ret = -EIO; - goto out; + goto stop; } - if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK, &buf) < 0) { - ret = -EIO; - goto out; - } - if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, - buf & ~DP_TEST_SINK_START) < 0) { - ret = -EIO; - goto out; - } -out: - hsw_enable_ips(intel_crtc); +stop: + intel_dp_sink_crc_stop(intel_dp); return ret; } @@ -4175,9 +4137,16 @@ static uint8_t intel_dp_autotest_edid(struct intel_dp *intel_dp) intel_dp->aux.i2c_defer_count); intel_dp->compliance_test_data = INTEL_DP_RESOLUTION_FAILSAFE; } else { + struct edid *block = intel_connector->detect_edid; + + /* We have to write the checksum + * of the last block read + */ + block += intel_connector->detect_edid->extensions; + if (!drm_dp_dpcd_write(&intel_dp->aux, DP_TEST_EDID_CHECKSUM, - &intel_connector->detect_edid->checksum, + &block->checksum, 1)) DRM_DEBUG_KMS("Failed to write EDID checksum\n"); @@ -4203,13 +4172,6 @@ static void intel_dp_handle_test_request(struct intel_dp *intel_dp) uint8_t rxdata = 0; int status = 0; - intel_dp->compliance_test_active = 0; - intel_dp->compliance_test_type = 0; - intel_dp->compliance_test_data = 0; - - intel_dp->aux.i2c_nack_count = 0; - intel_dp->aux.i2c_defer_count = 0; - status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_REQUEST, &rxdata, 1); if (status <= 0) { DRM_DEBUG_KMS("Could not read test request from sink\n"); @@ -4265,10 +4227,10 @@ go_again: if (bret == true) { /* check link status - esi[10] = 0x200c */ - if (intel_dp->active_mst_links && !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) { + if (intel_dp->active_mst_links && + !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) { DRM_DEBUG_KMS("channel EQ not ok, retraining\n"); intel_dp_start_link_train(intel_dp); - intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); } @@ -4325,10 +4287,15 @@ intel_dp_check_link_status(struct intel_dp *intel_dp) WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); - if (!intel_encoder->connectors_active) - return; + /* + * Clearing compliance test variables to allow capturing + * of values for next automated test request. + */ + intel_dp->compliance_test_active = 0; + intel_dp->compliance_test_type = 0; + intel_dp->compliance_test_data = 0; - if (WARN_ON(!intel_encoder->base.crtc)) + if (!intel_encoder->base.crtc) return; if (!to_intel_crtc(intel_encoder->base.crtc)->active) @@ -4358,11 +4325,12 @@ intel_dp_check_link_status(struct intel_dp *intel_dp) DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n"); } - if (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) { + /* if link training is requested we should perform it always */ + if ((intel_dp->compliance_test_type == DP_TEST_LINK_TRAINING) || + (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count))) { DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n", intel_encoder->base.name); intel_dp_start_link_train(intel_dp); - intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); } } @@ -4430,84 +4398,151 @@ edp_detect(struct intel_dp *intel_dp) return status; } -static enum drm_connector_status -ironlake_dp_detect(struct intel_dp *intel_dp) +static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + u32 bit; - if (!ibx_digital_port_connected(dev_priv, intel_dig_port)) - return connector_status_disconnected; + switch (port->port) { + case PORT_A: + return true; + case PORT_B: + bit = SDE_PORTB_HOTPLUG; + break; + case PORT_C: + bit = SDE_PORTC_HOTPLUG; + break; + case PORT_D: + bit = SDE_PORTD_HOTPLUG; + break; + default: + MISSING_CASE(port->port); + return false; + } - return intel_dp_detect_dpcd(intel_dp); + return I915_READ(SDEISR) & bit; } -static int g4x_digital_port_connected(struct drm_device *dev, - struct intel_digital_port *intel_dig_port) +static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port) { - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t bit; + u32 bit; - if (IS_VALLEYVIEW(dev)) { - switch (intel_dig_port->port) { - case PORT_B: - bit = PORTB_HOTPLUG_LIVE_STATUS_VLV; - break; - case PORT_C: - bit = PORTC_HOTPLUG_LIVE_STATUS_VLV; - break; - case PORT_D: - bit = PORTD_HOTPLUG_LIVE_STATUS_VLV; - break; - default: - return -EINVAL; - } - } else { - switch (intel_dig_port->port) { - case PORT_B: - bit = PORTB_HOTPLUG_LIVE_STATUS_G4X; - break; - case PORT_C: - bit = PORTC_HOTPLUG_LIVE_STATUS_G4X; - break; - case PORT_D: - bit = PORTD_HOTPLUG_LIVE_STATUS_G4X; - break; - default: - return -EINVAL; - } + switch (port->port) { + case PORT_A: + return true; + case PORT_B: + bit = SDE_PORTB_HOTPLUG_CPT; + break; + case PORT_C: + bit = SDE_PORTC_HOTPLUG_CPT; + break; + case PORT_D: + bit = SDE_PORTD_HOTPLUG_CPT; + break; + case PORT_E: + bit = SDE_PORTE_HOTPLUG_SPT; + break; + default: + MISSING_CASE(port->port); + return false; } - if ((I915_READ(PORT_HOTPLUG_STAT) & bit) == 0) - return 0; - return 1; + return I915_READ(SDEISR) & bit; } -static enum drm_connector_status -g4x_dp_detect(struct intel_dp *intel_dp) +static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - int ret; + u32 bit; - /* Can't disconnect eDP, but you can close the lid... */ - if (is_edp(intel_dp)) { - enum drm_connector_status status; + switch (port->port) { + case PORT_B: + bit = PORTB_HOTPLUG_LIVE_STATUS_G4X; + break; + case PORT_C: + bit = PORTC_HOTPLUG_LIVE_STATUS_G4X; + break; + case PORT_D: + bit = PORTD_HOTPLUG_LIVE_STATUS_G4X; + break; + default: + MISSING_CASE(port->port); + return false; + } - status = intel_panel_detect(dev); - if (status == connector_status_unknown) - status = connector_status_connected; - return status; + return I915_READ(PORT_HOTPLUG_STAT) & bit; +} + +static bool vlv_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port) +{ + u32 bit; + + switch (port->port) { + case PORT_B: + bit = PORTB_HOTPLUG_LIVE_STATUS_VLV; + break; + case PORT_C: + bit = PORTC_HOTPLUG_LIVE_STATUS_VLV; + break; + case PORT_D: + bit = PORTD_HOTPLUG_LIVE_STATUS_VLV; + break; + default: + MISSING_CASE(port->port); + return false; } - ret = g4x_digital_port_connected(dev, intel_dig_port); - if (ret == -EINVAL) - return connector_status_unknown; - else if (ret == 0) - return connector_status_disconnected; + return I915_READ(PORT_HOTPLUG_STAT) & bit; +} + +static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *intel_dig_port) +{ + struct intel_encoder *intel_encoder = &intel_dig_port->base; + enum port port; + u32 bit; + + intel_hpd_pin_to_port(intel_encoder->hpd_pin, &port); + switch (port) { + case PORT_A: + bit = BXT_DE_PORT_HP_DDIA; + break; + case PORT_B: + bit = BXT_DE_PORT_HP_DDIB; + break; + case PORT_C: + bit = BXT_DE_PORT_HP_DDIC; + break; + default: + MISSING_CASE(port); + return false; + } + + return I915_READ(GEN8_DE_PORT_ISR) & bit; +} - return intel_dp_detect_dpcd(intel_dp); +/* + * intel_digital_port_connected - is the specified port connected? + * @dev_priv: i915 private structure + * @port: the port to test + * + * Return %true if @port is connected, %false otherwise. + */ +bool intel_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port) +{ + if (HAS_PCH_IBX(dev_priv)) + return ibx_digital_port_connected(dev_priv, port); + if (HAS_PCH_SPLIT(dev_priv)) + return cpt_digital_port_connected(dev_priv, port); + else if (IS_BROXTON(dev_priv)) + return bxt_digital_port_connected(dev_priv, port); + else if (IS_VALLEYVIEW(dev_priv)) + return vlv_digital_port_connected(dev_priv, port); + else + return g4x_digital_port_connected(dev_priv, port); } static struct edid * @@ -4553,26 +4588,6 @@ intel_dp_unset_edid(struct intel_dp *intel_dp) intel_dp->has_audio = false; } -static enum intel_display_power_domain -intel_dp_power_get(struct intel_dp *dp) -{ - struct intel_encoder *encoder = &dp_to_dig_port(dp)->base; - enum intel_display_power_domain power_domain; - - power_domain = intel_display_port_power_domain(encoder); - intel_display_power_get(to_i915(encoder->base.dev), power_domain); - - return power_domain; -} - -static void -intel_dp_power_put(struct intel_dp *dp, - enum intel_display_power_domain power_domain) -{ - struct intel_encoder *encoder = &dp_to_dig_port(dp)->base; - intel_display_power_put(to_i915(encoder->base.dev), power_domain); -} - static enum drm_connector_status intel_dp_detect(struct drm_connector *connector, bool force) { @@ -4596,17 +4611,25 @@ intel_dp_detect(struct drm_connector *connector, bool force) return connector_status_disconnected; } - power_domain = intel_dp_power_get(intel_dp); + power_domain = intel_display_port_aux_power_domain(intel_encoder); + intel_display_power_get(to_i915(dev), power_domain); /* Can't disconnect eDP, but you can close the lid... */ if (is_edp(intel_dp)) status = edp_detect(intel_dp); - else if (HAS_PCH_SPLIT(dev)) - status = ironlake_dp_detect(intel_dp); + else if (intel_digital_port_connected(to_i915(dev), + dp_to_dig_port(intel_dp))) + status = intel_dp_detect_dpcd(intel_dp); else - status = g4x_dp_detect(intel_dp); - if (status != connector_status_connected) + status = connector_status_disconnected; + + if (status != connector_status_connected) { + intel_dp->compliance_test_active = 0; + intel_dp->compliance_test_type = 0; + intel_dp->compliance_test_data = 0; + goto out; + } intel_dp_probe_oui(intel_dp); @@ -4620,6 +4643,14 @@ intel_dp_detect(struct drm_connector *connector, bool force) goto out; } + /* + * Clearing NACK and defer counts to get their exact values + * while reading EDID which are required by Compliance tests + * 4.2.2.4 and 4.2.2.5 + */ + intel_dp->aux.i2c_nack_count = 0; + intel_dp->aux.i2c_defer_count = 0; + intel_dp_set_edid(intel_dp); if (intel_encoder->type != INTEL_OUTPUT_EDP) @@ -4641,7 +4672,7 @@ intel_dp_detect(struct drm_connector *connector, bool force) } out: - intel_dp_power_put(intel_dp, power_domain); + intel_display_power_put(to_i915(dev), power_domain); return status; } @@ -4650,6 +4681,7 @@ intel_dp_force(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev); enum intel_display_power_domain power_domain; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", @@ -4659,11 +4691,12 @@ intel_dp_force(struct drm_connector *connector) if (connector->status != connector_status_connected) return; - power_domain = intel_dp_power_get(intel_dp); + power_domain = intel_display_port_aux_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); intel_dp_set_edid(intel_dp); - intel_dp_power_put(intel_dp, power_domain); + intel_display_power_put(dev_priv, power_domain); if (intel_encoder->type != INTEL_OUTPUT_EDP) intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; @@ -4748,7 +4781,7 @@ intel_dp_set_property(struct drm_connector *connector, if (property == dev_priv->broadcast_rgb_property) { bool old_auto = intel_dp->color_range_auto; - uint32_t old_range = intel_dp->color_range; + bool old_range = intel_dp->limited_color_range; switch (val) { case INTEL_BROADCAST_RGB_AUTO: @@ -4756,18 +4789,18 @@ intel_dp_set_property(struct drm_connector *connector, break; case INTEL_BROADCAST_RGB_FULL: intel_dp->color_range_auto = false; - intel_dp->color_range = 0; + intel_dp->limited_color_range = false; break; case INTEL_BROADCAST_RGB_LIMITED: intel_dp->color_range_auto = false; - intel_dp->color_range = DP_COLOR_RANGE_16_235; + intel_dp->limited_color_range = true; break; default: return -EINVAL; } if (old_auto == intel_dp->color_range_auto && - old_range == intel_dp->color_range) + old_range == intel_dp->limited_color_range) return 0; goto done; @@ -4822,7 +4855,7 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder) struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_dp *intel_dp = &intel_dig_port->dp; - drm_dp_aux_unregister(&intel_dp->aux); + intel_dp_aux_fini(intel_dp); intel_dp_mst_encoder_cleanup(intel_dig_port); if (is_edp(intel_dp)) { cancel_delayed_work_sync(&intel_dp->panel_vdd_work); @@ -4879,7 +4912,7 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) * indefinitely. */ DRM_DEBUG_KMS("VDD left on by BIOS, adjusting state tracking\n"); - power_domain = intel_display_port_power_domain(&intel_dig_port->base); + power_domain = intel_display_port_aux_power_domain(&intel_dig_port->base); intel_display_power_get(dev_priv, power_domain); edp_panel_vdd_schedule_off(intel_dp); @@ -4909,7 +4942,7 @@ static void intel_dp_encoder_reset(struct drm_encoder *encoder) } static const struct drm_connector_funcs intel_dp_connector_funcs = { - .dpms = intel_connector_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_dp_detect, .force = intel_dp_force, .fill_modes = drm_helper_probe_single_connector_modes, @@ -4931,12 +4964,6 @@ static const struct drm_encoder_funcs intel_dp_enc_funcs = { .destroy = intel_dp_encoder_destroy, }; -void -intel_dp_hot_plug(struct intel_encoder *intel_encoder) -{ - return; -} - enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) { @@ -4947,7 +4974,8 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) enum intel_display_power_domain power_domain; enum irqreturn ret = IRQ_NONE; - if (intel_dig_port->base.type != INTEL_OUTPUT_EDP) + if (intel_dig_port->base.type != INTEL_OUTPUT_EDP && + intel_dig_port->base.type != INTEL_OUTPUT_HDMI) intel_dig_port->base.type = INTEL_OUTPUT_DISPLAYPORT; if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) { @@ -4966,20 +4994,15 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) port_name(intel_dig_port->port), long_hpd ? "long" : "short"); - power_domain = intel_display_port_power_domain(intel_encoder); + power_domain = intel_display_port_aux_power_domain(intel_encoder); intel_display_power_get(dev_priv, power_domain); if (long_hpd) { /* indicate that we need to restart link training */ intel_dp->train_set_valid = false; - if (HAS_PCH_SPLIT(dev)) { - if (!ibx_digital_port_connected(dev_priv, intel_dig_port)) - goto mst_fail; - } else { - if (g4x_digital_port_connected(dev, intel_dig_port) != 1) - goto mst_fail; - } + if (!intel_digital_port_connected(dev_priv, intel_dig_port)) + goto mst_fail; if (!intel_dp_get_dpcd(intel_dp)) { goto mst_fail; @@ -4987,9 +5010,12 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) intel_dp_probe_oui(intel_dp); - if (!intel_dp_probe_mst(intel_dp)) + if (!intel_dp_probe_mst(intel_dp)) { + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + intel_dp_check_link_status(intel_dp); + drm_modeset_unlock(&dev->mode_config.connection_mutex); goto mst_fail; - + } } else { if (intel_dp->is_mst) { if (intel_dp_check_mst_status(intel_dp) == -EINVAL) @@ -4997,10 +5023,6 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) } if (!intel_dp->is_mst) { - /* - * we'll check the link status via the normal hot plug path later - - * but for short hpds we should check it now - */ drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); intel_dp_check_link_status(intel_dp); drm_modeset_unlock(&dev->mode_config.connection_mutex); @@ -5023,37 +5045,26 @@ put_power: return ret; } -/* Return which DP Port should be selected for Transcoder DP control */ -int -intel_trans_dp_port_sel(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct intel_encoder *intel_encoder; - struct intel_dp *intel_dp; - - for_each_encoder_on_crtc(dev, crtc, intel_encoder) { - intel_dp = enc_to_intel_dp(&intel_encoder->base); - - if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT || - intel_encoder->type == INTEL_OUTPUT_EDP) - return intel_dp->output_reg; - } - - return -1; -} - -/* check the VBT to see whether the eDP is on DP-D port */ +/* check the VBT to see whether the eDP is on another port */ bool intel_dp_is_edp(struct drm_device *dev, enum port port) { struct drm_i915_private *dev_priv = dev->dev_private; union child_device_config *p_child; int i; static const short port_mapping[] = { - [PORT_B] = PORT_IDPB, - [PORT_C] = PORT_IDPC, - [PORT_D] = PORT_IDPD, + [PORT_B] = DVO_PORT_DPB, + [PORT_C] = DVO_PORT_DPC, + [PORT_D] = DVO_PORT_DPD, + [PORT_E] = DVO_PORT_DPE, }; + /* + * eDP not supported on g4x. so bail out early just + * for a bit extra safety in case the VBT is bonkers. + */ + if (INTEL_INFO(dev)->gen < 5) + return false; + if (port == PORT_A) return true; @@ -5104,8 +5115,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, struct drm_i915_private *dev_priv = dev->dev_private; struct edp_power_seq cur, vbt, spec, *final = &intel_dp->pps_delays; - u32 pp_on, pp_off, pp_div, pp; - int pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg; + u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0; + i915_reg_t pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg; lockdep_assert_held(&dev_priv->pps_mutex); @@ -5113,7 +5124,16 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, if (final->t11_t12 != 0) return; - if (HAS_PCH_SPLIT(dev)) { + if (IS_BROXTON(dev)) { + /* + * TODO: BXT has 2 sets of PPS registers. + * Correct Register for Broxton need to be identified + * using VBT. hardcoding for now + */ + pp_ctrl_reg = BXT_PP_CONTROL(0); + pp_on_reg = BXT_PP_ON_DELAYS(0); + pp_off_reg = BXT_PP_OFF_DELAYS(0); + } else if (HAS_PCH_SPLIT(dev)) { pp_ctrl_reg = PCH_PP_CONTROL; pp_on_reg = PCH_PP_ON_DELAYS; pp_off_reg = PCH_PP_OFF_DELAYS; @@ -5129,12 +5149,14 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, /* Workaround: Need to write PP_CONTROL with the unlock key as * the very first thing. */ - pp = ironlake_get_pp_control(intel_dp); - I915_WRITE(pp_ctrl_reg, pp); + pp_ctl = ironlake_get_pp_control(intel_dp); pp_on = I915_READ(pp_on_reg); pp_off = I915_READ(pp_off_reg); - pp_div = I915_READ(pp_div_reg); + if (!IS_BROXTON(dev)) { + I915_WRITE(pp_ctrl_reg, pp_ctl); + pp_div = I915_READ(pp_div_reg); + } /* Pull timing values out of registers */ cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >> @@ -5149,8 +5171,17 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >> PANEL_POWER_DOWN_DELAY_SHIFT; - cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >> + if (IS_BROXTON(dev)) { + u16 tmp = (pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >> + BXT_POWER_CYCLE_DELAY_SHIFT; + if (tmp > 0) + cur.t11_t12 = (tmp - 1) * 1000; + else + cur.t11_t12 = 0; + } else { + cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >> PANEL_POWER_CYCLE_DELAY_SHIFT) * 1000; + } DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12); @@ -5207,13 +5238,23 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, struct drm_i915_private *dev_priv = dev->dev_private; u32 pp_on, pp_off, pp_div, port_sel = 0; int div = HAS_PCH_SPLIT(dev) ? intel_pch_rawclk(dev) : intel_hrawclk(dev); - int pp_on_reg, pp_off_reg, pp_div_reg; + i915_reg_t pp_on_reg, pp_off_reg, pp_div_reg, pp_ctrl_reg; enum port port = dp_to_dig_port(intel_dp)->port; const struct edp_power_seq *seq = &intel_dp->pps_delays; lockdep_assert_held(&dev_priv->pps_mutex); - if (HAS_PCH_SPLIT(dev)) { + if (IS_BROXTON(dev)) { + /* + * TODO: BXT has 2 sets of PPS registers. + * Correct Register for Broxton need to be identified + * using VBT. hardcoding for now + */ + pp_ctrl_reg = BXT_PP_CONTROL(0); + pp_on_reg = BXT_PP_ON_DELAYS(0); + pp_off_reg = BXT_PP_OFF_DELAYS(0); + + } else if (HAS_PCH_SPLIT(dev)) { pp_on_reg = PCH_PP_ON_DELAYS; pp_off_reg = PCH_PP_OFF_DELAYS; pp_div_reg = PCH_PP_DIVISOR; @@ -5239,9 +5280,16 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT); /* Compute the divisor for the pp clock, simply match the Bspec * formula. */ - pp_div = ((100 * div)/2 - 1) << PP_REFERENCE_DIVIDER_SHIFT; - pp_div |= (DIV_ROUND_UP(seq->t11_t12, 1000) - << PANEL_POWER_CYCLE_DELAY_SHIFT); + if (IS_BROXTON(dev)) { + pp_div = I915_READ(pp_ctrl_reg); + pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK; + pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000) + << BXT_POWER_CYCLE_DELAY_SHIFT); + } else { + pp_div = ((100 * div)/2 - 1) << PP_REFERENCE_DIVIDER_SHIFT; + pp_div |= (DIV_ROUND_UP(seq->t11_t12, 1000) + << PANEL_POWER_CYCLE_DELAY_SHIFT); + } /* Haswell doesn't have any port selection bits for the panel * power sequencer any more. */ @@ -5258,11 +5306,16 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, I915_WRITE(pp_on_reg, pp_on); I915_WRITE(pp_off_reg, pp_off); - I915_WRITE(pp_div_reg, pp_div); + if (IS_BROXTON(dev)) + I915_WRITE(pp_ctrl_reg, pp_div); + else + I915_WRITE(pp_div_reg, pp_div); DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n", I915_READ(pp_on_reg), I915_READ(pp_off_reg), + IS_BROXTON(dev) ? + (I915_READ(pp_ctrl_reg) & BXT_POWER_CYCLE_DELAY_MASK) : I915_READ(pp_div_reg)); } @@ -5286,7 +5339,6 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate) struct intel_dp *intel_dp = dev_priv->drrs.dp; struct intel_crtc_state *config = NULL; struct intel_crtc *intel_crtc = NULL; - u32 reg, val; enum drrs_refresh_rate_type index = DRRS_HIGH_RR; if (refresh_rate <= 0) { @@ -5348,9 +5400,10 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate) DRM_ERROR("Unsupported refreshrate type\n"); } } else if (INTEL_INFO(dev)->gen > 6) { - reg = PIPECONF(intel_crtc->config->cpu_transcoder); - val = I915_READ(reg); + i915_reg_t reg = PIPECONF(intel_crtc->config->cpu_transcoder); + u32 val; + val = I915_READ(reg); if (index > DRRS_HIGH_RR) { if (IS_VALLEYVIEW(dev)) val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV; @@ -5467,13 +5520,12 @@ unlock: } /** - * intel_edp_drrs_invalidate - Invalidate DRRS + * intel_edp_drrs_invalidate - Disable Idleness DRRS * @dev: DRM device * @frontbuffer_bits: frontbuffer plane tracking bits * - * When there is a disturbance on screen (due to cursor movement/time - * update etc), DRRS needs to be invalidated, i.e. need to switch to - * high RR. + * This function gets called everytime rendering on the given planes start. + * Hence DRRS needs to be Upclocked, i.e. (LOW_RR -> HIGH_RR). * * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. */ @@ -5498,26 +5550,27 @@ void intel_edp_drrs_invalidate(struct drm_device *dev, crtc = dp_to_dig_port(dev_priv->drrs.dp)->base.base.crtc; pipe = to_intel_crtc(crtc)->pipe; - if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) { + frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); + dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits; + + /* invalidate means busy screen hence upclock */ + if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) intel_dp_set_drrs_state(dev_priv->dev, dev_priv->drrs.dp->attached_connector->panel. fixed_mode->vrefresh); - } - frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); - - dev_priv->drrs.busy_frontbuffer_bits |= frontbuffer_bits; mutex_unlock(&dev_priv->drrs.mutex); } /** - * intel_edp_drrs_flush - Flush DRRS + * intel_edp_drrs_flush - Restart Idleness DRRS * @dev: DRM device * @frontbuffer_bits: frontbuffer plane tracking bits * - * When there is no movement on screen, DRRS work can be scheduled. - * This DRRS work is responsible for setting relevant registers after a - * timeout of 1 second. + * This function gets called every time rendering on the given planes has + * completed or flip on a crtc is completed. So DRRS should be upclocked + * (LOW_RR -> HIGH_RR). And also Idleness detection should be started again, + * if no other planes are dirty. * * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits. */ @@ -5541,10 +5594,21 @@ void intel_edp_drrs_flush(struct drm_device *dev, crtc = dp_to_dig_port(dev_priv->drrs.dp)->base.base.crtc; pipe = to_intel_crtc(crtc)->pipe; + + frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); dev_priv->drrs.busy_frontbuffer_bits &= ~frontbuffer_bits; - if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR && - !dev_priv->drrs.busy_frontbuffer_bits) + /* flush means busy screen hence upclock */ + if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) + intel_dp_set_drrs_state(dev_priv->dev, + dev_priv->drrs.dp->attached_connector->panel. + fixed_mode->vrefresh); + + /* + * flush also means no more activity hence schedule downclock, if all + * other fbs are quiescent too + */ + if (!dev_priv->drrs.busy_frontbuffer_bits) schedule_delayed_work(&dev_priv->drrs.work, msecs_to_jiffies(1000)); mutex_unlock(&dev_priv->drrs.mutex); @@ -5738,7 +5802,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, } intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); - intel_connector->panel.backlight_power = intel_edp_backlight_power; + intel_connector->panel.backlight.power = intel_edp_backlight_power; intel_panel_setup_backlight(connector, pipe); return true; @@ -5754,7 +5818,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum port port = intel_dig_port->port; - int type; + int type, ret; intel_dp->pps_pipe = INVALID_PIPE; @@ -5775,6 +5839,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, else intel_dp->get_aux_send_ctl = i9xx_get_aux_send_ctl; + if (HAS_DDI(dev)) + intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain; + /* Preserve the current hw state. */ intel_dp->DP = I915_READ(intel_dp->output_reg); intel_dp->attached_connector = intel_connector; @@ -5826,6 +5893,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, break; case PORT_B: intel_encoder->hpd_pin = HPD_PORT_B; + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + intel_encoder->hpd_pin = HPD_PORT_A; break; case PORT_C: intel_encoder->hpd_pin = HPD_PORT_C; @@ -5833,6 +5902,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, case PORT_D: intel_encoder->hpd_pin = HPD_PORT_D; break; + case PORT_E: + intel_encoder->hpd_pin = HPD_PORT_E; + break; default: BUG(); } @@ -5847,7 +5919,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, pps_unlock(intel_dp); } - intel_dp_aux_init(intel_dp, intel_connector); + ret = intel_dp_aux_init(intel_dp, intel_connector); + if (ret) + goto fail; /* init MST on ports that can support it */ if (HAS_DP_MST(dev) && @@ -5856,20 +5930,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, intel_connector->base.base.id); if (!intel_edp_init_connector(intel_dp, intel_connector)) { - drm_dp_aux_unregister(&intel_dp->aux); - if (is_edp(intel_dp)) { - cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - /* - * vdd might still be enabled do to the delayed vdd off. - * Make sure vdd is actually turned off here. - */ - pps_lock(intel_dp); - edp_panel_vdd_off_sync(intel_dp); - pps_unlock(intel_dp); - } - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - return false; + intel_dp_aux_fini(intel_dp); + intel_dp_mst_encoder_cleanup(intel_dig_port); + goto fail; } intel_dp_add_properties(intel_dp, connector); @@ -5886,10 +5949,27 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, i915_debugfs_connector_add(connector); return true; + +fail: + if (is_edp(intel_dp)) { + cancel_delayed_work_sync(&intel_dp->panel_vdd_work); + /* + * vdd might still be enabled do to the delayed vdd off. + * Make sure vdd is actually turned off here. + */ + pps_lock(intel_dp); + edp_panel_vdd_off_sync(intel_dp); + pps_unlock(intel_dp); + } + drm_connector_unregister(connector); + drm_connector_cleanup(connector); + + return false; } void -intel_dp_init(struct drm_device *dev, int output_reg, enum port port) +intel_dp_init(struct drm_device *dev, + i915_reg_t output_reg, enum port port) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_digital_port *intel_dig_port; @@ -5902,16 +5982,14 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) return; intel_connector = intel_connector_alloc(); - if (!intel_connector) { - kfree(intel_dig_port); - return; - } + if (!intel_connector) + goto err_connector_alloc; intel_encoder = &intel_dig_port->base; encoder = &intel_encoder->base; drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs, - DRM_MODE_ENCODER_TMDS); + DRM_MODE_ENCODER_TMDS, NULL); intel_encoder->compute_config = intel_dp_compute_config; intel_encoder->disable = intel_disable_dp; @@ -5923,6 +6001,7 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) intel_encoder->pre_enable = chv_pre_enable_dp; intel_encoder->enable = vlv_enable_dp; intel_encoder->post_disable = chv_post_disable_dp; + intel_encoder->post_pll_disable = chv_dp_post_pll_disable; } else if (IS_VALLEYVIEW(dev)) { intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable; intel_encoder->pre_enable = vlv_pre_enable_dp; @@ -5948,16 +6027,22 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); } intel_encoder->cloneable = 0; - intel_encoder->hot_plug = intel_dp_hot_plug; intel_dig_port->hpd_pulse = intel_dp_hpd_pulse; - dev_priv->hpd_irq_port[port] = intel_dig_port; + dev_priv->hotplug.irq_port[port] = intel_dig_port; - if (!intel_dp_init_connector(intel_dig_port, intel_connector)) { - drm_encoder_cleanup(encoder); - kfree(intel_dig_port); - kfree(intel_connector); - } + if (!intel_dp_init_connector(intel_dig_port, intel_connector)) + goto err_init_connector; + + return; + +err_init_connector: + drm_encoder_cleanup(encoder); + kfree(intel_connector); +err_connector_alloc: + kfree(intel_dig_port); + + return; } void intel_dp_mst_suspend(struct drm_device *dev) @@ -5967,7 +6052,7 @@ void intel_dp_mst_suspend(struct drm_device *dev) /* disable MST */ for (i = 0; i < I915_MAX_PORTS; i++) { - struct intel_digital_port *intel_dig_port = dev_priv->hpd_irq_port[i]; + struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i]; if (!intel_dig_port) continue; @@ -5986,7 +6071,7 @@ void intel_dp_mst_resume(struct drm_device *dev) int i; for (i = 0; i < I915_MAX_PORTS; i++) { - struct intel_digital_port *intel_dig_port = dev_priv->hpd_irq_port[i]; + struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i]; if (!intel_dig_port) continue; if (intel_dig_port->base.type == INTEL_OUTPUT_DISPLAYPORT) { diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c new file mode 100644 index 000000000000..88887938e0bf --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dp_link_training.c @@ -0,0 +1,323 @@ +/* + * Copyright © 2008-2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "intel_drv.h" + +static void +intel_get_adjust_train(struct intel_dp *intel_dp, + const uint8_t link_status[DP_LINK_STATUS_SIZE]) +{ + uint8_t v = 0; + uint8_t p = 0; + int lane; + uint8_t voltage_max; + uint8_t preemph_max; + + for (lane = 0; lane < intel_dp->lane_count; lane++) { + uint8_t this_v = drm_dp_get_adjust_request_voltage(link_status, lane); + uint8_t this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); + + if (this_v > v) + v = this_v; + if (this_p > p) + p = this_p; + } + + voltage_max = intel_dp_voltage_max(intel_dp); + if (v >= voltage_max) + v = voltage_max | DP_TRAIN_MAX_SWING_REACHED; + + preemph_max = intel_dp_pre_emphasis_max(intel_dp, v); + if (p >= preemph_max) + p = preemph_max | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED; + + for (lane = 0; lane < 4; lane++) + intel_dp->train_set[lane] = v | p; +} + +static bool +intel_dp_set_link_train(struct intel_dp *intel_dp, + uint8_t dp_train_pat) +{ + uint8_t buf[sizeof(intel_dp->train_set) + 1]; + int ret, len; + + intel_dp_program_link_training_pattern(intel_dp, dp_train_pat); + + buf[0] = dp_train_pat; + if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) == + DP_TRAINING_PATTERN_DISABLE) { + /* don't write DP_TRAINING_LANEx_SET on disable */ + len = 1; + } else { + /* DP_TRAINING_LANEx_SET follow DP_TRAINING_PATTERN_SET */ + memcpy(buf + 1, intel_dp->train_set, intel_dp->lane_count); + len = intel_dp->lane_count + 1; + } + + ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET, + buf, len); + + return ret == len; +} + +static bool +intel_dp_reset_link_train(struct intel_dp *intel_dp, + uint8_t dp_train_pat) +{ + if (!intel_dp->train_set_valid) + memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); + intel_dp_set_signal_levels(intel_dp); + return intel_dp_set_link_train(intel_dp, dp_train_pat); +} + +static bool +intel_dp_update_link_train(struct intel_dp *intel_dp) +{ + int ret; + + intel_dp_set_signal_levels(intel_dp); + + ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET, + intel_dp->train_set, intel_dp->lane_count); + + return ret == intel_dp->lane_count; +} + +/* Enable corresponding port and start training pattern 1 */ +static void +intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) +{ + int i; + uint8_t voltage; + int voltage_tries, loop_tries; + uint8_t link_config[2]; + uint8_t link_bw, rate_select; + + if (intel_dp->prepare_link_retrain) + intel_dp->prepare_link_retrain(intel_dp); + + intel_dp_compute_rate(intel_dp, intel_dp->link_rate, + &link_bw, &rate_select); + + /* Write the link configuration data */ + link_config[0] = link_bw; + link_config[1] = intel_dp->lane_count; + if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) + link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN; + drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2); + if (intel_dp->num_sink_rates) + drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET, + &rate_select, 1); + + link_config[0] = 0; + link_config[1] = DP_SET_ANSI_8B10B; + drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2); + + intel_dp->DP |= DP_PORT_EN; + + /* clock recovery */ + if (!intel_dp_reset_link_train(intel_dp, + DP_TRAINING_PATTERN_1 | + DP_LINK_SCRAMBLING_DISABLE)) { + DRM_ERROR("failed to enable link training\n"); + return; + } + + voltage = 0xff; + voltage_tries = 0; + loop_tries = 0; + for (;;) { + uint8_t link_status[DP_LINK_STATUS_SIZE]; + + drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd); + if (!intel_dp_get_link_status(intel_dp, link_status)) { + DRM_ERROR("failed to get link status\n"); + break; + } + + if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { + DRM_DEBUG_KMS("clock recovery OK\n"); + break; + } + + /* + * if we used previously trained voltage and pre-emphasis values + * and we don't get clock recovery, reset link training values + */ + if (intel_dp->train_set_valid) { + DRM_DEBUG_KMS("clock recovery not ok, reset"); + /* clear the flag as we are not reusing train set */ + intel_dp->train_set_valid = false; + if (!intel_dp_reset_link_train(intel_dp, + DP_TRAINING_PATTERN_1 | + DP_LINK_SCRAMBLING_DISABLE)) { + DRM_ERROR("failed to enable link training\n"); + return; + } + continue; + } + + /* Check to see if we've tried the max voltage */ + for (i = 0; i < intel_dp->lane_count; i++) + if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0) + break; + if (i == intel_dp->lane_count) { + ++loop_tries; + if (loop_tries == 5) { + DRM_ERROR("too many full retries, give up\n"); + break; + } + intel_dp_reset_link_train(intel_dp, + DP_TRAINING_PATTERN_1 | + DP_LINK_SCRAMBLING_DISABLE); + voltage_tries = 0; + continue; + } + + /* Check to see if we've tried the same voltage 5 times */ + if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) { + ++voltage_tries; + if (voltage_tries == 5) { + DRM_ERROR("too many voltage retries, give up\n"); + break; + } + } else + voltage_tries = 0; + voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; + + /* Update training set as requested by target */ + intel_get_adjust_train(intel_dp, link_status); + if (!intel_dp_update_link_train(intel_dp)) { + DRM_ERROR("failed to update link training\n"); + break; + } + } +} + +static void +intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) +{ + bool channel_eq = false; + int tries, cr_tries; + uint32_t training_pattern = DP_TRAINING_PATTERN_2; + + /* + * Training Pattern 3 for HBR2 or 1.2 devices that support it. + * + * Intel platforms that support HBR2 also support TPS3. TPS3 support is + * also mandatory for downstream devices that support HBR2. + * + * Due to WaDisableHBR2 SKL < B0 is the only exception where TPS3 is + * supported but still not enabled. + */ + if (intel_dp_source_supports_hbr2(intel_dp) && + drm_dp_tps3_supported(intel_dp->dpcd)) + training_pattern = DP_TRAINING_PATTERN_3; + else if (intel_dp->link_rate == 540000) + DRM_ERROR("5.4 Gbps link rate without HBR2/TPS3 support\n"); + + /* channel equalization */ + if (!intel_dp_set_link_train(intel_dp, + training_pattern | + DP_LINK_SCRAMBLING_DISABLE)) { + DRM_ERROR("failed to start channel equalization\n"); + return; + } + + tries = 0; + cr_tries = 0; + channel_eq = false; + for (;;) { + uint8_t link_status[DP_LINK_STATUS_SIZE]; + + if (cr_tries > 5) { + DRM_ERROR("failed to train DP, aborting\n"); + break; + } + + drm_dp_link_train_channel_eq_delay(intel_dp->dpcd); + if (!intel_dp_get_link_status(intel_dp, link_status)) { + DRM_ERROR("failed to get link status\n"); + break; + } + + /* Make sure clock is still ok */ + if (!drm_dp_clock_recovery_ok(link_status, + intel_dp->lane_count)) { + intel_dp->train_set_valid = false; + intel_dp_link_training_clock_recovery(intel_dp); + intel_dp_set_link_train(intel_dp, + training_pattern | + DP_LINK_SCRAMBLING_DISABLE); + cr_tries++; + continue; + } + + if (drm_dp_channel_eq_ok(link_status, + intel_dp->lane_count)) { + channel_eq = true; + break; + } + + /* Try 5 times, then try clock recovery if that fails */ + if (tries > 5) { + intel_dp->train_set_valid = false; + intel_dp_link_training_clock_recovery(intel_dp); + intel_dp_set_link_train(intel_dp, + training_pattern | + DP_LINK_SCRAMBLING_DISABLE); + tries = 0; + cr_tries++; + continue; + } + + /* Update training set as requested by target */ + intel_get_adjust_train(intel_dp, link_status); + if (!intel_dp_update_link_train(intel_dp)) { + DRM_ERROR("failed to update link training\n"); + break; + } + ++tries; + } + + intel_dp_set_idle_link_train(intel_dp); + + if (channel_eq) { + intel_dp->train_set_valid = true; + DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n"); + } +} + +void intel_dp_stop_link_train(struct intel_dp *intel_dp) +{ + intel_dp_set_link_train(intel_dp, + DP_TRAINING_PATTERN_DISABLE); +} + +void +intel_dp_start_link_train(struct intel_dp *intel_dp) +{ + intel_dp_link_training_clock_recovery(intel_dp); + intel_dp_link_training_channel_equalization(intel_dp); +} diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c index 600afdbef8c9..e8d369d0a713 100644 --- a/drivers/gpu/drm/i915/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/intel_dp_mst.c @@ -33,13 +33,14 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { + struct drm_device *dev = encoder->base.dev; struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base); struct intel_digital_port *intel_dig_port = intel_mst->primary; struct intel_dp *intel_dp = &intel_dig_port->dp; struct drm_atomic_state *state; int bpp, i; - int lane_count, slots, rate; - struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + int lane_count, slots; + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct drm_connector *drm_connector; struct intel_connector *connector, *found = NULL; struct drm_connector_state *connector_state; @@ -55,20 +56,11 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, */ lane_count = drm_dp_max_lane_count(intel_dp->dpcd); - rate = intel_dp_max_link_rate(intel_dp); - - if (intel_dp->num_sink_rates) { - intel_dp->link_bw = 0; - intel_dp->rate_select = intel_dp_rate_select(intel_dp, rate); - } else { - intel_dp->link_bw = drm_dp_link_rate_to_bw_code(rate); - intel_dp->rate_select = 0; - } - intel_dp->lane_count = lane_count; + pipe_config->lane_count = lane_count; pipe_config->pipe_bpp = 24; - pipe_config->port_clock = rate; + pipe_config->port_clock = intel_dp_max_link_rate(intel_dp); state = pipe_config->base.state; @@ -86,7 +78,7 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, return false; } - mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->clock, bpp); + mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp); pipe_config->pbn = mst_pbn; slots = drm_dp_find_vcpi_slots(&intel_dp->mst_mgr, mst_pbn); @@ -97,6 +89,10 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, &pipe_config->dp_m_n); pipe_config->dp_m_n.tu = slots; + + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + hsw_dp_set_ddi_pll_sel(pipe_config); + return true; } @@ -168,24 +164,24 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder) return; } + /* MST encoders are bound to a crtc, not to a connector, + * force the mapping here for get_hw_state. + */ + found->encoder = encoder; + DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links); intel_mst->port = found->port; if (intel_dp->active_mst_links == 0) { - enum port port = intel_ddi_get_encoder_port(encoder); + intel_ddi_clk_select(encoder, intel_crtc->config); - /* FIXME: add support for SKL */ - if (INTEL_INFO(dev)->gen < 9) - I915_WRITE(PORT_CLK_SEL(port), - intel_crtc->config->ddi_pll_sel); + intel_dp_set_link_params(intel_dp, intel_crtc->config); intel_ddi_init_dp_buf_reg(&intel_dig_port->base); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); - intel_dp_start_link_train(intel_dp); - intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); } @@ -276,6 +272,10 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder, break; } pipe_config->base.adjusted_mode.flags |= flags; + + pipe_config->lane_count = + ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1; + intel_dp_get_m_n(crtc, pipe_config); intel_ddi_clock_get(&intel_dig_port->base, pipe_config); @@ -328,7 +328,7 @@ intel_dp_mst_connector_destroy(struct drm_connector *connector) } static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { - .dpms = intel_connector_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_dp_mst_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_dp_mst_set_property, @@ -395,7 +395,7 @@ static const struct drm_encoder_funcs intel_dp_mst_enc_funcs = { static bool intel_dp_mst_get_hw_state(struct intel_connector *connector) { - if (connector->encoder) { + if (connector->encoder && connector->base.state->crtc) { enum pipe pipe; if (!connector->encoder->get_hw_state(connector->encoder, &pipe)) return false; @@ -406,17 +406,23 @@ static bool intel_dp_mst_get_hw_state(struct intel_connector *connector) static void intel_connector_add_to_fbdev(struct intel_connector *connector) { -#ifdef CONFIG_DRM_I915_FBDEV +#ifdef CONFIG_DRM_FBDEV_EMULATION struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper, &connector->base); + + if (dev_priv->fbdev) + drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper, + &connector->base); #endif } static void intel_connector_remove_from_fbdev(struct intel_connector *connector) { -#ifdef CONFIG_DRM_I915_FBDEV +#ifdef CONFIG_DRM_FBDEV_EMULATION struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper, &connector->base); + + if (dev_priv->fbdev) + drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper, + &connector->base); #endif } @@ -452,12 +458,17 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0); drm_mode_connector_set_path_property(connector, pathprop); - drm_reinit_primary_mode_group(dev); - mutex_lock(&dev->mode_config.mutex); + return connector; +} + +static void intel_dp_register_mst_connector(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + struct drm_device *dev = connector->dev; + drm_modeset_lock_all(dev); intel_connector_add_to_fbdev(intel_connector); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_unlock_all(dev); drm_connector_register(&intel_connector->base); - return connector; } static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, @@ -465,19 +476,28 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, { struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; + /* need to nuke the connector */ - mutex_lock(&dev->mode_config.mutex); - intel_connector_dpms(connector, DRM_MODE_DPMS_OFF); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_lock_all(dev); + if (connector->state->crtc) { + struct drm_mode_set set; + int ret; + + memset(&set, 0, sizeof(set)); + set.crtc = connector->state->crtc, + + ret = drm_atomic_helper_set_config(&set); + + WARN(ret, "Disabling mst crtc failed with %i\n", ret); + } + drm_modeset_unlock_all(dev); intel_connector->unregister(intel_connector); - mutex_lock(&dev->mode_config.mutex); + drm_modeset_lock_all(dev); intel_connector_remove_from_fbdev(intel_connector); drm_connector_cleanup(connector); - mutex_unlock(&dev->mode_config.mutex); - - drm_reinit_primary_mode_group(dev); + drm_modeset_unlock_all(dev); kfree(intel_connector); DRM_DEBUG_KMS("\n"); @@ -494,6 +514,7 @@ static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr) static struct drm_dp_mst_topology_cbs mst_cbs = { .add_connector = intel_dp_add_mst_connector, + .register_connector = intel_dp_register_mst_connector, .destroy_connector = intel_dp_destroy_mst_connector, .hotplug = intel_dp_mst_hotplug, }; @@ -515,7 +536,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum intel_mst->primary = intel_dig_port; drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs, - DRM_MODE_ENCODER_DPMST); + DRM_MODE_ENCODER_DPMST, NULL); intel_encoder->type = INTEL_OUTPUT_DP_MST; intel_encoder->crtc_mask = 0x7; diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 105928382e21..50f83d220249 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -123,22 +123,14 @@ struct intel_framebuffer { struct intel_fbdev { struct drm_fb_helper helper; struct intel_framebuffer *fb; - struct list_head fbdev_list; - struct drm_display_mode *our_mode; int preferred_bpp; }; struct intel_encoder { struct drm_encoder base; - /* - * The new crtc this encoder will be driven from. Only differs from - * base->crtc while a modeset is in progress. - */ - struct intel_crtc *new_crtc; enum intel_output_type type; unsigned int cloneable; - bool connectors_active; void (*hot_plug)(struct intel_encoder *); bool (*compute_config)(struct intel_encoder *, struct intel_crtc_state *); @@ -148,6 +140,7 @@ struct intel_encoder { void (*mode_set)(struct intel_encoder *intel_encoder); void (*disable)(struct intel_encoder *); void (*post_disable)(struct intel_encoder *); + void (*post_pll_disable)(struct intel_encoder *); /* Read out the current hw state of this connector, returning true if * the encoder is active. If the encoder is enabled it also set the pipe * it is connected to in the pipe parameter. */ @@ -182,10 +175,24 @@ struct intel_panel { bool enabled; bool combination_mode; /* gen 2/4 only */ bool active_low_pwm; + + /* PWM chip */ + bool util_pin_active_low; /* bxt+ */ + u8 controller; /* bxt+ only */ + struct pwm_device *pwm; + struct backlight_device *device; - } backlight; - void (*backlight_power)(struct intel_connector *, bool enable); + /* Connector and platform specific backlight functions */ + int (*setup)(struct intel_connector *connector, enum pipe pipe); + uint32_t (*get)(struct intel_connector *connector); + void (*set)(struct intel_connector *connector, uint32_t level); + void (*disable)(struct intel_connector *connector); + void (*enable)(struct intel_connector *connector); + uint32_t (*hz_to_pwm)(struct intel_connector *connector, + uint32_t hz); + void (*power)(struct intel_connector *, bool enable); + } backlight; }; struct intel_connector { @@ -195,12 +202,6 @@ struct intel_connector { */ struct intel_encoder *encoder; - /* - * The new encoder this connector will be driven. Only differs from - * encoder while a modeset is in progress. - */ - struct intel_encoder *new_encoder; - /* Reads out the current hw, returning true if the connector is enabled * and active (i.e. dpms ON state). */ bool (*get_hw_state)(struct intel_connector *); @@ -241,6 +242,15 @@ typedef struct dpll { int p; } intel_clock_t; +struct intel_atomic_state { + struct drm_atomic_state base; + + unsigned int cdclk; + bool dpll_set; + struct intel_shared_dpll_config shared_dpll[I915_NUM_PLLS]; + struct intel_wm_config wm_config; +}; + struct intel_plane_state { struct drm_plane_state base; struct drm_rect src; @@ -256,7 +266,7 @@ struct intel_plane_state { * plane requiring a scaler: * - During check_plane, its bit is set in * crtc_state->scaler_state.scaler_users by calling helper function - * update_scaler_users. + * update_scaler_plane. * - scaler_id indicates the scaler it got assigned. * * plane doesn't require a scaler: @@ -264,9 +274,14 @@ struct intel_plane_state { * got disabled. * - During check_plane, corresponding bit is reset in * crtc_state->scaler_state.scaler_users by calling helper function - * update_scaler_users. + * update_scaler_plane. */ int scaler_id; + + struct drm_intel_sprite_colorkey ckey; + + /* async flip related structures */ + struct drm_i915_gem_request *wait_req; }; struct intel_initial_plane_config { @@ -286,7 +301,6 @@ struct intel_initial_plane_config { #define SKL_MAX_DST_H 4096 struct intel_scaler { - int id; int in_use; uint32_t mode; }; @@ -319,6 +333,24 @@ struct intel_crtc_scaler_state { int scaler_id; }; +/* drm_mode->private_flags */ +#define I915_MODE_FLAG_INHERITED 1 + +struct intel_pipe_wm { + struct intel_wm_level wm[5]; + uint32_t linetime; + bool fbc_wm_enabled; + bool pipe_enabled; + bool sprites_enabled; + bool sprites_scaled; +}; + +struct skl_pipe_wm { + struct skl_wm_level wm[8]; + struct skl_wm_level trans_wm; + uint32_t linetime; +}; + struct intel_crtc_state { struct drm_crtc_state base; @@ -331,9 +363,10 @@ struct intel_crtc_state { * accordingly. */ #define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */ -#define PIPE_CONFIG_QUIRK_INHERITED_MODE (1<<1) /* mode inherited from firmware */ unsigned long quirks; + bool update_pipe; + /* Pipe source size (ie. panel fitter input size) * All planes will be positioned inside this space, * and get clipped at the edges. */ @@ -360,6 +393,9 @@ struct intel_crtc_state { * accordingly. */ bool has_dp_encoder; + /* DSI has special cases */ + bool has_dsi_encoder; + /* Whether we should send NULL infoframes. Required for audio. */ bool has_hdmi_sink; @@ -420,6 +456,8 @@ struct intel_crtc_state { /* Used by SDVO (and if we ever fix it, HDMI). */ unsigned pixel_multiplier; + uint8_t lane_count; + /* Panel fitter controls for gen2-gen4 + VLV */ struct { u32 control; @@ -447,15 +485,32 @@ struct intel_crtc_state { int pbn; struct intel_crtc_scaler_state scaler_state; + + /* w/a for waiting 2 vblanks during crtc enable */ + enum pipe hsw_workaround_pipe; + + /* IVB sprite scaling w/a (WaCxSRDisabledForSpriteScaling:ivb) */ + bool disable_lp_wm; + + struct { + /* + * optimal watermarks, programmed post-vblank when this state + * is committed + */ + union { + struct intel_pipe_wm ilk; + struct skl_pipe_wm skl; + } optimal; + } wm; }; -struct intel_pipe_wm { - struct intel_wm_level wm[5]; - uint32_t linetime; - bool fbc_wm_enabled; - bool pipe_enabled; - bool sprites_enabled; - bool sprites_scaled; +struct vlv_wm_state { + struct vlv_pipe_wm wm[3]; + struct vlv_sr_wm sr[3]; + uint8_t num_active_planes; + uint8_t num_levels; + uint8_t level; + bool cxsr; }; struct intel_mmio_flip { @@ -463,12 +518,7 @@ struct intel_mmio_flip { struct drm_i915_private *i915; struct drm_i915_gem_request *req; struct intel_crtc *crtc; -}; - -struct skl_pipe_wm { - struct skl_wm_level wm[8]; - struct skl_wm_level trans_wm; - uint32_t linetime; + unsigned int rotation; }; /* @@ -478,17 +528,12 @@ struct skl_pipe_wm { * and thus can't be run with interrupts disabled. */ struct intel_crtc_atomic_commit { - /* vblank evasion */ - bool evade; - unsigned start_vbl_count; - /* Sleepable operations to perform before commit */ - bool wait_for_flips; bool disable_fbc; bool disable_ips; + bool disable_cxsr; bool pre_disable_primary; - bool update_wm; - unsigned disabled_planes; + bool update_wm_pre, update_wm_post; /* Sleepable operations to perform after commit */ unsigned fb_bits; @@ -520,6 +565,8 @@ struct intel_crtc { * gen4+ this only adjusts up to a tile, offsets within a tile are * handled in the hw itself (with the TILEOFF register). */ unsigned long dspaddr_offset; + int adjusted_x; + int adjusted_y; struct drm_i915_gem_object *cursor_bo; uint32_t cursor_addr; @@ -527,9 +574,7 @@ struct intel_crtc { uint32_t cursor_size; uint32_t cursor_base; - struct intel_initial_plane_config plane_config; struct intel_crtc_state *config; - bool new_enabled; /* reset counter value when the last flip was submitted */ unsigned int reset_counter; @@ -541,17 +586,29 @@ struct intel_crtc { /* per-pipe watermark state */ struct { /* watermarks currently being used */ - struct intel_pipe_wm active; - /* SKL wm values currently in use */ - struct skl_pipe_wm skl_active; + union { + struct intel_pipe_wm ilk; + struct skl_pipe_wm skl; + } active; + /* allow CxSR on this pipe */ + bool cxsr_allowed; } wm; int scanline_offset; + struct { + unsigned start_vbl_count; + ktime_t start_vbl_time; + int min_vbl, max_vbl; + int scanline_start; + } debug; + struct intel_crtc_atomic_commit atomic; /* scalers available on this crtc */ int num_scalers; + + struct vlv_wm_state wm_state; }; struct intel_plane_wm_parameters { @@ -570,6 +627,7 @@ struct intel_plane_wm_parameters { bool scaled; u64 tiling; unsigned int rotation; + uint16_t fifo_size; }; struct intel_plane { @@ -578,9 +636,7 @@ struct intel_plane { enum pipe pipe; bool can_scale; int max_downscale; - - /* FIXME convert to properties */ - struct drm_intel_sprite_colorkey ckey; + uint32_t frontbuffer_bit; /* Since we need to change the watermarks before/after * enabling/disabling the planes, we need to store the parameters here @@ -603,8 +659,9 @@ struct intel_plane { uint32_t x, uint32_t y, uint32_t src_w, uint32_t src_h); void (*disable_plane)(struct drm_plane *plane, - struct drm_crtc *crtc, bool force); + struct drm_crtc *crtc); int (*check_plane)(struct drm_plane *plane, + struct intel_crtc_state *crtc_state, struct intel_plane_state *state); void (*commit_plane)(struct drm_plane *plane, struct intel_plane_state *state); @@ -629,6 +686,7 @@ struct cxsr_latency { unsigned long cursor_hpll_disable; }; +#define to_intel_atomic_state(x) container_of(x, struct intel_atomic_state, base) #define to_intel_crtc(x) container_of(x, struct intel_crtc, base) #define to_intel_crtc_state(x) container_of(x, struct intel_crtc_state, base) #define to_intel_connector(x) container_of(x, struct intel_connector, base) @@ -639,22 +697,24 @@ struct cxsr_latency { #define intel_fb_obj(x) (x ? to_intel_framebuffer(x)->obj : NULL) struct intel_hdmi { - u32 hdmi_reg; + i915_reg_t hdmi_reg; int ddc_bus; - uint32_t color_range; + bool limited_color_range; bool color_range_auto; bool has_hdmi_sink; bool has_audio; enum hdmi_force_audio force_audio; bool rgb_quant_range_selectable; enum hdmi_picture_aspect aspect_ratio; + struct intel_connector *attached_connector; void (*write_infoframe)(struct drm_encoder *encoder, enum hdmi_infoframe_type type, const void *frame, ssize_t len); void (*set_infoframes)(struct drm_encoder *encoder, bool enable, - struct drm_display_mode *adjusted_mode); - bool (*infoframe_enabled)(struct drm_encoder *encoder); + const struct drm_display_mode *adjusted_mode); + bool (*infoframe_enabled)(struct drm_encoder *encoder, + const struct intel_crtc_state *pipe_config); }; struct intel_dp_mst_encoder; @@ -681,16 +741,16 @@ enum link_m_n_set { }; struct intel_dp { - uint32_t output_reg; - uint32_t aux_ch_ctl_reg; + i915_reg_t output_reg; + i915_reg_t aux_ch_ctl_reg; + i915_reg_t aux_ch_data_reg[5]; uint32_t DP; + int link_rate; + uint8_t lane_count; bool has_audio; enum hdmi_force_audio force_audio; - uint32_t color_range; + bool limited_color_range; bool color_range_auto; - uint8_t link_bw; - uint8_t rate_select; - uint8_t lane_count; uint8_t dpcd[DP_RECEIVER_CAP_SIZE]; uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; @@ -719,7 +779,6 @@ struct intel_dp { enum pipe pps_pipe; struct edp_power_seq pps_delays; - bool use_tps3; bool can_mst; /* this port supports mst */ bool is_mst; int active_mst_links; @@ -739,6 +798,10 @@ struct intel_dp { bool has_aux_irq, int send_bytes, uint32_t aux_clock_divider); + + /* This is called before a link training is starterd */ + void (*prepare_link_retrain)(struct intel_dp *intel_dp); + bool train_set_valid; /* Displayport compliance testing */ @@ -754,6 +817,7 @@ struct intel_digital_port { struct intel_dp dp; struct intel_hdmi hdmi; enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool); + bool release_cl2_override; }; struct intel_dp_mst_encoder { @@ -763,7 +827,7 @@ struct intel_dp_mst_encoder { void *port; /* store this opaque as its illegal to dereference it */ }; -static inline int +static inline enum dpio_channel vlv_dport_to_channel(struct intel_digital_port *dport) { switch (dport->port) { @@ -777,7 +841,21 @@ vlv_dport_to_channel(struct intel_digital_port *dport) } } -static inline int +static inline enum dpio_phy +vlv_dport_to_phy(struct intel_digital_port *dport) +{ + switch (dport->port) { + case PORT_B: + case PORT_C: + return DPIO_PHY0; + case PORT_D: + return DPIO_PHY1; + default: + BUG(); + } +} + +static inline enum dpio_channel vlv_pipe_to_channel(enum pipe pipe) { switch (pipe) { @@ -818,8 +896,8 @@ struct intel_unpin_work { u32 flip_count; u32 gtt_offset; struct drm_i915_gem_request *flip_queued_req; - int flip_queued_vblank; - int flip_ready_vblank; + u32 flip_queued_vblank; + u32 flip_ready_vblank; bool enable_stall_check; }; @@ -883,7 +961,8 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe); void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum transcoder pch_transcoder); -void i9xx_check_fifo_underruns(struct drm_i915_private *dev_priv); +void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv); +void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv); /* i915_irq.c */ void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); @@ -914,6 +993,8 @@ void intel_crt_init(struct drm_device *dev); /* intel_ddi.c */ +void intel_ddi_clk_select(struct intel_encoder *encoder, + const struct intel_crtc_state *pipe_config); void intel_prepare_ddi(struct drm_device *dev); void hsw_fdi_link_train(struct drm_crtc *crtc); void intel_ddi_init(struct drm_device *dev, enum port port); @@ -928,7 +1009,7 @@ void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc); bool intel_ddi_pll_select(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state); void intel_ddi_set_pipe_settings(struct drm_crtc *crtc); -void intel_ddi_prepare_link_retrain(struct drm_encoder *encoder); +void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp); bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector); void intel_ddi_fdi_disable(struct drm_crtc *crtc); void intel_ddi_get_config(struct intel_encoder *encoder, @@ -940,43 +1021,23 @@ void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder); void intel_ddi_clock_get(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config); void intel_ddi_set_vc_payload_alloc(struct drm_crtc *crtc, bool state); -void bxt_ddi_vswing_sequence(struct drm_device *dev, u32 level, - enum port port, int type); +uint32_t ddi_signal_levels(struct intel_dp *intel_dp); /* intel_frontbuffer.c */ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, enum fb_op_origin origin); void intel_frontbuffer_flip_prepare(struct drm_device *dev, unsigned frontbuffer_bits); void intel_frontbuffer_flip_complete(struct drm_device *dev, unsigned frontbuffer_bits); -void intel_frontbuffer_flush(struct drm_device *dev, - unsigned frontbuffer_bits); -/** - * intel_frontbuffer_flip - synchronous frontbuffer flip - * @dev: DRM device - * @frontbuffer_bits: frontbuffer plane tracking bits - * - * This function gets called after scheduling a flip on @obj. This is for - * synchronous plane updates which will happen on the next vblank and which will - * not get delayed by pending gpu rendering. - * - * Can be called without any locks held. - */ -static inline void intel_frontbuffer_flip(struct drm_device *dev, - unsigned frontbuffer_bits) -{ - intel_frontbuffer_flush(dev, frontbuffer_bits); -} - + unsigned frontbuffer_bits); unsigned int intel_fb_align_height(struct drm_device *dev, unsigned int height, uint32_t pixel_format, uint64_t fb_format_modifier); -void intel_fb_obj_flush(struct drm_i915_gem_object *obj, bool retire); - +void intel_fb_obj_flush(struct drm_i915_gem_object *obj, bool retire, + enum fb_op_origin origin); u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier, uint32_t pixel_format); @@ -991,20 +1052,15 @@ void i915_audio_component_cleanup(struct drm_i915_private *dev_priv); extern const struct drm_plane_funcs intel_plane_funcs; bool intel_has_pending_fb_unpin(struct drm_device *dev); int intel_pch_rawclk(struct drm_device *dev); +int intel_hrawclk(struct drm_device *dev); void intel_mark_busy(struct drm_device *dev); void intel_mark_idle(struct drm_device *dev); void intel_crtc_restore_mode(struct drm_crtc *crtc); -void intel_crtc_control(struct drm_crtc *crtc, bool enable); -void intel_crtc_reset(struct intel_crtc *crtc); -void intel_crtc_update_dpms(struct drm_crtc *crtc); +int intel_display_suspend(struct drm_device *dev); void intel_encoder_destroy(struct drm_encoder *encoder); int intel_connector_init(struct intel_connector *); struct intel_connector *intel_connector_alloc(void); -void intel_connector_dpms(struct drm_connector *, int mode); bool intel_connector_get_hw_state(struct intel_connector *connector); -void intel_modeset_check_state(struct drm_device *dev); -bool ibx_digital_port_connected(struct drm_i915_private *dev_priv, - struct intel_digital_port *port); void intel_connector_attach_encoder(struct intel_connector *connector, struct intel_encoder *encoder); struct drm_encoder *intel_best_encoder(struct drm_connector *connector); @@ -1021,6 +1077,15 @@ intel_wait_for_vblank(struct drm_device *dev, int pipe) { drm_wait_one_vblank(dev, pipe); } +static inline void +intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe) +{ + const struct intel_crtc *crtc = + to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + + if (crtc->active) + intel_wait_for_vblank(dev, pipe); +} int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp); void vlv_wait_port_ready(struct drm_i915_private *dev_priv, struct intel_digital_port *dport, @@ -1034,8 +1099,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx); int intel_pin_and_fence_fb_obj(struct drm_plane *plane, struct drm_framebuffer *fb, - const struct drm_plane_state *plane_state, - struct intel_engine_cs *pipelined); + const struct drm_plane_state *plane_state); struct drm_framebuffer * __intel_framebuffer_create(struct drm_device *dev, struct drm_mode_fb_cmd2 *mode_cmd, @@ -1045,10 +1109,8 @@ void intel_finish_page_flip(struct drm_device *dev, int pipe); void intel_finish_page_flip_plane(struct drm_device *dev, int plane); void intel_check_page_flip(struct drm_device *dev, int pipe); int intel_prepare_plane_fb(struct drm_plane *plane, - struct drm_framebuffer *fb, const struct drm_plane_state *new_state); void intel_cleanup_plane_fb(struct drm_plane *plane, - struct drm_framebuffer *fb, const struct drm_plane_state *old_state); int intel_plane_atomic_get_property(struct drm_plane *plane, const struct drm_plane_state *state, @@ -1058,10 +1120,12 @@ int intel_plane_atomic_set_property(struct drm_plane *plane, struct drm_plane_state *state, struct drm_property *property, uint64_t val); +int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state, + struct drm_plane_state *plane_state); unsigned int intel_tile_height(struct drm_device *dev, uint32_t pixel_format, - uint64_t fb_format_modifier); + uint64_t fb_format_modifier, unsigned int plane); static inline bool intel_rotation_90_or_270(unsigned int rotation) @@ -1072,9 +1136,6 @@ intel_rotation_90_or_270(unsigned int rotation) void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane); -bool intel_wm_need_update(struct drm_plane *plane, - struct drm_plane_state *state); - /* shared dpll functions */ struct intel_shared_dpll *intel_crtc_to_shared_dpll(struct intel_crtc *crtc); void assert_shared_dpll(struct drm_i915_private *dev_priv, @@ -1084,7 +1145,6 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv, #define assert_shared_dpll_disabled(d, p) assert_shared_dpll(d, p, false) struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc, struct intel_crtc_state *state); -void intel_put_shared_dpll(struct intel_crtc *crtc); void vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, const struct dpll *dpll); @@ -1104,7 +1164,8 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, void assert_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, bool state); #define assert_pipe_enabled(d, p) assert_pipe(d, p, true) #define assert_pipe_disabled(d, p) assert_pipe(d, p, false) -unsigned long intel_gen4_compute_page_offset(int *x, int *y, +unsigned long intel_gen4_compute_page_offset(struct drm_i915_private *dev_priv, + int *x, int *y, unsigned int tiling_mode, unsigned int bpp, unsigned int pitch); @@ -1114,13 +1175,15 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv); void hsw_disable_pc8(struct drm_i915_private *dev_priv); void broxton_init_cdclk(struct drm_device *dev); void broxton_uninit_cdclk(struct drm_device *dev); -void broxton_set_cdclk(struct drm_device *dev, int frequency); void broxton_ddi_phy_init(struct drm_device *dev); void broxton_ddi_phy_uninit(struct drm_device *dev); void bxt_enable_dc9(struct drm_i915_private *dev_priv); void bxt_disable_dc9(struct drm_i915_private *dev_priv); void skl_init_cdclk(struct drm_i915_private *dev_priv); +int skl_sanitize_cdclk(struct drm_i915_private *dev_priv); void skl_uninit_cdclk(struct drm_i915_private *dev_priv); +void skl_enable_dc6(struct drm_i915_private *dev_priv); +void skl_disable_dc6(struct drm_i915_private *dev_priv); void intel_dp_get_m_n(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n); @@ -1130,42 +1193,42 @@ ironlake_check_encoder_dotclock(const struct intel_crtc_state *pipe_config, int dotclock); bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, intel_clock_t *best_clock); +int chv_calc_dpll_params(int refclk, intel_clock_t *pll_clock); + bool intel_crtc_active(struct drm_crtc *crtc); void hsw_enable_ips(struct intel_crtc *crtc); void hsw_disable_ips(struct intel_crtc *crtc); enum intel_display_power_domain intel_display_port_power_domain(struct intel_encoder *intel_encoder); +enum intel_display_power_domain +intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder); void intel_mode_from_pipe_config(struct drm_display_mode *mode, struct intel_crtc_state *pipe_config); -void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc); void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file); -void skl_detach_scalers(struct intel_crtc *intel_crtc); -int skl_update_scaler_users(struct intel_crtc *intel_crtc, - struct intel_crtc_state *crtc_state, struct intel_plane *intel_plane, - struct intel_plane_state *plane_state, int force_detach); + +int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state); int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state); -unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane, - struct drm_i915_gem_object *obj); +u32 intel_plane_obj_offset(struct intel_plane *intel_plane, + struct drm_i915_gem_object *obj, + unsigned int plane); + u32 skl_plane_ctl_format(uint32_t pixel_format); u32 skl_plane_ctl_tiling(uint64_t fb_modifier); u32 skl_plane_ctl_rotation(unsigned int rotation); /* intel_csr.c */ -void intel_csr_ucode_init(struct drm_device *dev); -enum csr_state intel_csr_load_status_get(struct drm_i915_private *dev_priv); -void intel_csr_load_status_set(struct drm_i915_private *dev_priv, - enum csr_state state); -void intel_csr_load_program(struct drm_device *dev); -void intel_csr_ucode_fini(struct drm_device *dev); -void assert_csr_loaded(struct drm_i915_private *dev_priv); +void intel_csr_ucode_init(struct drm_i915_private *); +void intel_csr_load_program(struct drm_i915_private *); +void intel_csr_ucode_fini(struct drm_i915_private *); /* intel_dp.c */ -void intel_dp_init(struct drm_device *dev, int output_reg, enum port port); +void intel_dp_init(struct drm_device *dev, i915_reg_t output_reg, enum port port); bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector); +void intel_dp_set_link_params(struct intel_dp *intel_dp, + const struct intel_crtc_state *pipe_config); void intel_dp_start_link_train(struct intel_dp *intel_dp); -void intel_dp_complete_link_train(struct intel_dp *intel_dp); void intel_dp_stop_link_train(struct intel_dp *intel_dp); void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode); void intel_dp_encoder_destroy(struct drm_encoder *encoder); @@ -1194,6 +1257,25 @@ void intel_edp_drrs_disable(struct intel_dp *intel_dp); void intel_edp_drrs_invalidate(struct drm_device *dev, unsigned frontbuffer_bits); void intel_edp_drrs_flush(struct drm_device *dev, unsigned frontbuffer_bits); +bool intel_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port); +void hsw_dp_set_ddi_pll_sel(struct intel_crtc_state *pipe_config); + +void +intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, + uint8_t dp_train_pat); +void +intel_dp_set_signal_levels(struct intel_dp *intel_dp); +void intel_dp_set_idle_link_train(struct intel_dp *intel_dp); +uint8_t +intel_dp_voltage_max(struct intel_dp *intel_dp); +uint8_t +intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing); +void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock, + uint8_t *link_bw, uint8_t *rate_select); +bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp); +bool +intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]); /* intel_dp_mst.c */ int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id); @@ -1207,9 +1289,9 @@ void intel_dvo_init(struct drm_device *dev); /* legacy fbdev emulation in intel_fbdev.c */ -#ifdef CONFIG_DRM_I915_FBDEV +#ifdef CONFIG_DRM_FBDEV_EMULATION extern int intel_fbdev_init(struct drm_device *dev); -extern void intel_fbdev_initial_config(void *data, async_cookie_t cookie); +extern void intel_fbdev_initial_config_async(struct drm_device *dev); extern void intel_fbdev_fini(struct drm_device *dev); extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous); extern void intel_fbdev_output_poll_changed(struct drm_device *dev); @@ -1220,7 +1302,7 @@ static inline int intel_fbdev_init(struct drm_device *dev) return 0; } -static inline void intel_fbdev_initial_config(void *data, async_cookie_t cookie) +static inline void intel_fbdev_initial_config_async(struct drm_device *dev) { } @@ -1238,18 +1320,22 @@ static inline void intel_fbdev_restore_mode(struct drm_device *dev) #endif /* intel_fbc.c */ -bool intel_fbc_enabled(struct drm_device *dev); -void intel_fbc_update(struct drm_device *dev); +bool intel_fbc_is_active(struct drm_i915_private *dev_priv); +void intel_fbc_deactivate(struct intel_crtc *crtc); +void intel_fbc_update(struct intel_crtc *crtc); void intel_fbc_init(struct drm_i915_private *dev_priv); -void intel_fbc_disable(struct drm_device *dev); +void intel_fbc_enable(struct intel_crtc *crtc); +void intel_fbc_disable(struct drm_i915_private *dev_priv); +void intel_fbc_disable_crtc(struct intel_crtc *crtc); void intel_fbc_invalidate(struct drm_i915_private *dev_priv, unsigned int frontbuffer_bits, enum fb_op_origin origin); void intel_fbc_flush(struct drm_i915_private *dev_priv, - unsigned int frontbuffer_bits); + unsigned int frontbuffer_bits, enum fb_op_origin origin); +void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv); /* intel_hdmi.c */ -void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port); +void intel_hdmi_init(struct drm_device *dev, i915_reg_t hdmi_reg, enum port port); void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector); struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder); @@ -1268,6 +1354,7 @@ int intel_connector_update_modes(struct drm_connector *connector, int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter); void intel_attach_force_audio_property(struct drm_connector *connector); void intel_attach_broadcast_rgb_property(struct drm_connector *connector); +void intel_attach_aspect_ratio_property(struct drm_connector *connector); /* intel_overlay.c */ @@ -1300,7 +1387,6 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) void intel_panel_enable_backlight(struct intel_connector *connector); void intel_panel_disable_backlight(struct intel_connector *connector); void intel_panel_destroy_backlight(struct drm_connector *connector); -void intel_panel_init_backlight_funcs(struct drm_device *dev); enum drm_connector_status intel_panel_detect(struct drm_device *dev); extern struct drm_display_mode *intel_find_panel_downclock( struct drm_device *dev, @@ -1314,17 +1400,24 @@ void intel_backlight_unregister(struct drm_device *dev); void intel_psr_enable(struct intel_dp *intel_dp); void intel_psr_disable(struct intel_dp *intel_dp); void intel_psr_invalidate(struct drm_device *dev, - unsigned frontbuffer_bits); + unsigned frontbuffer_bits); void intel_psr_flush(struct drm_device *dev, - unsigned frontbuffer_bits); + unsigned frontbuffer_bits, + enum fb_op_origin origin); void intel_psr_init(struct drm_device *dev); -void intel_psr_single_frame_update(struct drm_device *dev); +void intel_psr_single_frame_update(struct drm_device *dev, + unsigned frontbuffer_bits); /* intel_runtime_pm.c */ int intel_power_domains_init(struct drm_i915_private *); void intel_power_domains_fini(struct drm_i915_private *); -void intel_power_domains_init_hw(struct drm_i915_private *dev_priv); +void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume); +void intel_power_domains_suspend(struct drm_i915_private *dev_priv); +void skl_pw1_misc_io_init(struct drm_i915_private *dev_priv); +void skl_pw1_misc_io_fini(struct drm_i915_private *dev_priv); void intel_runtime_pm_enable(struct drm_i915_private *dev_priv); +const char * +intel_display_power_domain_str(enum intel_display_power_domain domain); bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); @@ -1334,25 +1427,23 @@ void intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); void intel_display_power_put(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); -void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv); -void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv); void intel_runtime_pm_get(struct drm_i915_private *dev_priv); void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv); void intel_runtime_pm_put(struct drm_i915_private *dev_priv); void intel_display_set_init_power(struct drm_i915_private *dev, bool enable); +void chv_phy_powergate_lanes(struct intel_encoder *encoder, + bool override, unsigned int mask); +bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, + enum dpio_channel ch, bool override); + + /* intel_pm.c */ void intel_init_clock_gating(struct drm_device *dev); void intel_suspend_hw(struct drm_device *dev); int ilk_wm_max_level(const struct drm_device *dev); void intel_update_watermarks(struct drm_crtc *crtc); -void intel_update_sprite_watermarks(struct drm_plane *plane, - struct drm_crtc *crtc, - uint32_t sprite_width, - uint32_t sprite_height, - int pixel_size, - bool enabled, bool scaled); void intel_init_pm(struct drm_device *dev); void intel_pm_setup(struct drm_device *dev); void intel_gpu_ips_init(struct drm_i915_private *dev_priv); @@ -1372,34 +1463,29 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv, unsigned long submitted); void intel_queue_rps_boost_for_request(struct drm_device *dev, struct drm_i915_gem_request *req); +void vlv_wm_get_hw_state(struct drm_device *dev); void ilk_wm_get_hw_state(struct drm_device *dev); void skl_wm_get_hw_state(struct drm_device *dev); void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, struct skl_ddb_allocation *ddb /* out */); - +uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config); /* intel_sdvo.c */ -bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob); +bool intel_sdvo_init(struct drm_device *dev, + i915_reg_t reg, enum port port); /* intel_sprite.c */ int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane); -int intel_plane_restore(struct drm_plane *plane); int intel_sprite_set_colorkey(struct drm_device *dev, void *data, struct drm_file *file_priv); -bool intel_pipe_update_start(struct intel_crtc *crtc, - uint32_t *start_vbl_count); -void intel_pipe_update_end(struct intel_crtc *crtc, u32 start_vbl_count); +void intel_pipe_update_start(struct intel_crtc *crtc); +void intel_pipe_update_end(struct intel_crtc *crtc); /* intel_tv.c */ void intel_tv_init(struct drm_device *dev); /* intel_atomic.c */ -int intel_atomic_check(struct drm_device *dev, - struct drm_atomic_state *state); -int intel_atomic_commit(struct drm_device *dev, - struct drm_atomic_state *state, - bool async); int intel_connector_atomic_get_property(struct drm_connector *connector, const struct drm_connector_state *state, struct drm_property *property, @@ -1407,6 +1493,11 @@ int intel_connector_atomic_get_property(struct drm_connector *connector, struct drm_crtc_state *intel_crtc_duplicate_state(struct drm_crtc *crtc); void intel_crtc_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *state); +struct drm_atomic_state *intel_atomic_state_alloc(struct drm_device *dev); +void intel_atomic_state_clear(struct drm_atomic_state *); +struct intel_shared_dpll_config * +intel_atomic_get_shared_dpll_state(struct drm_atomic_state *s); + static inline struct intel_crtc_state * intel_atomic_get_crtc_state(struct drm_atomic_state *state, struct intel_crtc *crtc) diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c index b5a5558ecd63..fff9a66c32a1 100644 --- a/drivers/gpu/drm/i915/intel_dsi.c +++ b/drivers/gpu/drm/i915/intel_dsi.c @@ -31,6 +31,7 @@ #include <drm/drm_panel.h> #include <drm/drm_mipi_dsi.h> #include <linux/slab.h> +#include <linux/gpio/consumer.h> #include "i915_drv.h" #include "intel_drv.h" #include "intel_dsi.h" @@ -59,7 +60,8 @@ static void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port) DRM_ERROR("DPI FIFOs are not empty\n"); } -static void write_data(struct drm_i915_private *dev_priv, u32 reg, +static void write_data(struct drm_i915_private *dev_priv, + i915_reg_t reg, const u8 *data, u32 len) { u32 i, j; @@ -74,7 +76,8 @@ static void write_data(struct drm_i915_private *dev_priv, u32 reg, } } -static void read_data(struct drm_i915_private *dev_priv, u32 reg, +static void read_data(struct drm_i915_private *dev_priv, + i915_reg_t reg, u8 *data, u32 len) { u32 i, j; @@ -97,7 +100,8 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host, struct mipi_dsi_packet packet; ssize_t ret; const u8 *header, *data; - u32 data_reg, data_mask, ctrl_reg, ctrl_mask; + i915_reg_t data_reg, ctrl_reg; + u32 data_mask, ctrl_mask; ret = mipi_dsi_create_packet(&packet, msg); if (ret < 0) @@ -261,22 +265,19 @@ static inline bool is_cmd_mode(struct intel_dsi *intel_dsi) return intel_dsi->operation_mode == INTEL_DSI_COMMAND_MODE; } -static void intel_dsi_hot_plug(struct intel_encoder *encoder) -{ - DRM_DEBUG_KMS("\n"); -} - static bool intel_dsi_compute_config(struct intel_encoder *encoder, - struct intel_crtc_state *config) + struct intel_crtc_state *pipe_config) { struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, base); struct intel_connector *intel_connector = intel_dsi->attached_connector; struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; - struct drm_display_mode *adjusted_mode = &config->base.adjusted_mode; + struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; DRM_DEBUG_KMS("\n"); + pipe_config->has_dsi_encoder = true; + if (fixed_mode) intel_fixed_panel_mode(fixed_mode, adjusted_mode); @@ -286,58 +287,46 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, return true; } -static void intel_dsi_port_enable(struct intel_encoder *encoder) +static void bxt_dsi_device_ready(struct intel_encoder *encoder) { - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; - u32 temp; + u32 val; - if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) { - temp = I915_READ(VLV_CHICKEN_3); - temp &= ~PIXEL_OVERLAP_CNT_MASK | - intel_dsi->pixel_overlap << - PIXEL_OVERLAP_CNT_SHIFT; - I915_WRITE(VLV_CHICKEN_3, temp); - } + DRM_DEBUG_KMS("\n"); + /* Exit Low power state in 4 steps*/ for_each_dsi_port(port, intel_dsi->ports) { - temp = I915_READ(MIPI_PORT_CTRL(port)); - temp &= ~LANE_CONFIGURATION_MASK; - temp &= ~DUAL_LINK_MODE_MASK; - if (intel_dsi->ports == ((1 << PORT_A) | (1 << PORT_C))) { - temp |= (intel_dsi->dual_link - 1) - << DUAL_LINK_MODE_SHIFT; - temp |= intel_crtc->pipe ? - LANE_CONFIGURATION_DUAL_LINK_B : - LANE_CONFIGURATION_DUAL_LINK_A; - } - /* assert ip_tg_enable signal */ - I915_WRITE(MIPI_PORT_CTRL(port), temp | DPI_ENABLE); - POSTING_READ(MIPI_PORT_CTRL(port)); - } -} + /* 1. Enable MIPI PHY transparent latch */ + val = I915_READ(BXT_MIPI_PORT_CTRL(port)); + I915_WRITE(BXT_MIPI_PORT_CTRL(port), val | LP_OUTPUT_HOLD); + usleep_range(2000, 2500); -static void intel_dsi_port_disable(struct intel_encoder *encoder) -{ - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); - enum port port; - u32 temp; + /* 2. Enter ULPS */ + val = I915_READ(MIPI_DEVICE_READY(port)); + val &= ~ULPS_STATE_MASK; + val |= (ULPS_STATE_ENTER | DEVICE_READY); + I915_WRITE(MIPI_DEVICE_READY(port), val); + usleep_range(2, 3); + + /* 3. Exit ULPS */ + val = I915_READ(MIPI_DEVICE_READY(port)); + val &= ~ULPS_STATE_MASK; + val |= (ULPS_STATE_EXIT | DEVICE_READY); + I915_WRITE(MIPI_DEVICE_READY(port), val); + usleep_range(1000, 1500); - for_each_dsi_port(port, intel_dsi->ports) { - /* de-assert ip_tg_enable signal */ - temp = I915_READ(MIPI_PORT_CTRL(port)); - I915_WRITE(MIPI_PORT_CTRL(port), temp & ~DPI_ENABLE); - POSTING_READ(MIPI_PORT_CTRL(port)); + /* Clear ULPS and set device ready */ + val = I915_READ(MIPI_DEVICE_READY(port)); + val &= ~ULPS_STATE_MASK; + val |= DEVICE_READY; + I915_WRITE(MIPI_DEVICE_READY(port), val); } } -static void intel_dsi_device_ready(struct intel_encoder *encoder) +static void vlv_dsi_device_ready(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); @@ -376,6 +365,76 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder) } } +static void intel_dsi_device_ready(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + + if (IS_VALLEYVIEW(dev)) + vlv_dsi_device_ready(encoder); + else if (IS_BROXTON(dev)) + bxt_dsi_device_ready(encoder); +} + +static void intel_dsi_port_enable(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + + if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) { + u32 temp; + + temp = I915_READ(VLV_CHICKEN_3); + temp &= ~PIXEL_OVERLAP_CNT_MASK | + intel_dsi->pixel_overlap << + PIXEL_OVERLAP_CNT_SHIFT; + I915_WRITE(VLV_CHICKEN_3, temp); + } + + for_each_dsi_port(port, intel_dsi->ports) { + i915_reg_t port_ctrl = IS_BROXTON(dev) ? + BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port); + u32 temp; + + temp = I915_READ(port_ctrl); + + temp &= ~LANE_CONFIGURATION_MASK; + temp &= ~DUAL_LINK_MODE_MASK; + + if (intel_dsi->ports == ((1 << PORT_A) | (1 << PORT_C))) { + temp |= (intel_dsi->dual_link - 1) + << DUAL_LINK_MODE_SHIFT; + temp |= intel_crtc->pipe ? + LANE_CONFIGURATION_DUAL_LINK_B : + LANE_CONFIGURATION_DUAL_LINK_A; + } + /* assert ip_tg_enable signal */ + I915_WRITE(port_ctrl, temp | DPI_ENABLE); + POSTING_READ(port_ctrl); + } +} + +static void intel_dsi_port_disable(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + + for_each_dsi_port(port, intel_dsi->ports) { + i915_reg_t port_ctrl = IS_BROXTON(dev) ? + BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port); + u32 temp; + + /* de-assert ip_tg_enable signal */ + temp = I915_READ(port_ctrl); + I915_WRITE(port_ctrl, temp & ~DPI_ENABLE); + POSTING_READ(port_ctrl); + } +} + static void intel_dsi_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; @@ -401,8 +460,12 @@ static void intel_dsi_enable(struct intel_encoder *encoder) intel_dsi_port_enable(encoder); } + + intel_panel_enable_backlight(intel_dsi->attached_connector); } +static void intel_dsi_prepare(struct intel_encoder *intel_encoder); + static void intel_dsi_pre_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; @@ -415,25 +478,37 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder) DRM_DEBUG_KMS("\n"); - /* Disable DPOunit clock gating, can stall pipe - * and we need DPLL REFA always enabled */ - tmp = I915_READ(DPLL(pipe)); - tmp |= DPLL_REFA_CLK_ENABLE_VLV; - I915_WRITE(DPLL(pipe), tmp); + intel_dsi_prepare(encoder); + intel_enable_dsi_pll(encoder); - /* update the hw state for DPLL */ - intel_crtc->config->dpll_hw_state.dpll = DPLL_INTEGRATED_CLOCK_VLV | - DPLL_REFA_CLK_ENABLE_VLV; + /* Panel Enable over CRC PMIC */ + if (intel_dsi->gpio_panel) + gpiod_set_value_cansleep(intel_dsi->gpio_panel, 1); - tmp = I915_READ(DSPCLK_GATE_D); - tmp |= DPOUNIT_CLOCK_GATE_DISABLE; - I915_WRITE(DSPCLK_GATE_D, tmp); + msleep(intel_dsi->panel_on_delay); + + if (IS_VALLEYVIEW(dev)) { + /* + * Disable DPOunit clock gating, can stall pipe + * and we need DPLL REFA always enabled + */ + tmp = I915_READ(DPLL(pipe)); + tmp |= DPLL_REF_CLK_ENABLE_VLV; + I915_WRITE(DPLL(pipe), tmp); + + /* update the hw state for DPLL */ + intel_crtc->config->dpll_hw_state.dpll = + DPLL_INTEGRATED_REF_CLK_VLV | + DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; + + tmp = I915_READ(DSPCLK_GATE_D); + tmp |= DPOUNIT_CLOCK_GATE_DISABLE; + I915_WRITE(DSPCLK_GATE_D, tmp); + } /* put device in ready state */ intel_dsi_device_ready(encoder); - msleep(intel_dsi->panel_on_delay); - drm_panel_prepare(intel_dsi->panel); for_each_dsi_port(port, intel_dsi->ports) @@ -461,6 +536,8 @@ static void intel_dsi_pre_disable(struct intel_encoder *encoder) DRM_DEBUG_KMS("\n"); + intel_panel_disable_backlight(intel_dsi->attached_connector); + if (is_vid_mode(intel_dsi)) { /* Send Shutdown command to the panel in LP mode */ for_each_dsi_port(port, intel_dsi->ports) @@ -491,12 +568,7 @@ static void intel_dsi_disable(struct intel_encoder *encoder) /* Panel commands can be sent when clock is in LP11 */ I915_WRITE(MIPI_DEVICE_READY(port), 0x0); - temp = I915_READ(MIPI_CTRL(port)); - temp &= ~ESCAPE_CLOCK_DIVIDER_MASK; - I915_WRITE(MIPI_CTRL(port), temp | - intel_dsi->escape_clk_div << - ESCAPE_CLOCK_DIVIDER_SHIFT); - + intel_dsi_reset_clocks(encoder, port); I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP); temp = I915_READ(MIPI_DSI_FUNC_PRG(port)); @@ -515,13 +587,17 @@ static void intel_dsi_disable(struct intel_encoder *encoder) static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) { + struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; - u32 val; DRM_DEBUG_KMS("\n"); for_each_dsi_port(port, intel_dsi->ports) { + /* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */ + i915_reg_t port_ctrl = IS_BROXTON(dev) ? + BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A); + u32 val; I915_WRITE(MIPI_DEVICE_READY(port), DEVICE_READY | ULPS_STATE_ENTER); @@ -538,22 +614,20 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) /* Wait till Clock lanes are in LP-00 state for MIPI Port A * only. MIPI Port C has no similar bit for checking */ - if (wait_for(((I915_READ(MIPI_PORT_CTRL(PORT_A)) & AFE_LATCHOUT) - == 0x00000), 30)) + if (wait_for(((I915_READ(port_ctrl) & AFE_LATCHOUT) + == 0x00000), 30)) DRM_ERROR("DSI LP not going Low\n"); - /* Disable MIPI PHY transparent latch - * Common bit for both MIPI Port A & MIPI Port C - */ - val = I915_READ(MIPI_PORT_CTRL(PORT_A)); - I915_WRITE(MIPI_PORT_CTRL(PORT_A), val & ~LP_OUTPUT_HOLD); + /* Disable MIPI PHY transparent latch */ + val = I915_READ(port_ctrl); + I915_WRITE(port_ctrl, val & ~LP_OUTPUT_HOLD); usleep_range(1000, 1500); I915_WRITE(MIPI_DEVICE_READY(port), 0x00); usleep_range(2000, 2500); } - vlv_disable_dsi_pll(encoder); + intel_disable_dsi_pll(encoder); } static void intel_dsi_post_disable(struct intel_encoder *encoder) @@ -576,6 +650,10 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder) msleep(intel_dsi->panel_off_delay); msleep(intel_dsi->panel_pwr_cycle_delay); + + /* Panel Disable over CRC PMIC */ + if (intel_dsi->gpio_panel) + gpiod_set_value_cansleep(intel_dsi->gpio_panel, 0); } static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, @@ -585,7 +663,6 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); struct drm_device *dev = encoder->base.dev; enum intel_display_power_domain power_domain; - u32 dpi_enabled, func; enum port port; DRM_DEBUG_KMS("\n"); @@ -596,9 +673,12 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, /* XXX: this only works for one DSI output */ for_each_dsi_port(port, intel_dsi->ports) { + i915_reg_t ctrl_reg = IS_BROXTON(dev) ? + BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port); + u32 dpi_enabled, func; + func = I915_READ(MIPI_DSI_FUNC_PRG(port)); - dpi_enabled = I915_READ(MIPI_PORT_CTRL(port)) & - DPI_ENABLE; + dpi_enabled = I915_READ(ctrl_reg) & DPI_ENABLE; /* Due to some hardware limitations on BYT, MIPI Port C DPI * Enable bit does not get set. To check whether DSI Port C @@ -623,16 +703,22 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, static void intel_dsi_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - u32 pclk; + u32 pclk = 0; DRM_DEBUG_KMS("\n"); + pipe_config->has_dsi_encoder = true; + /* * DPLL_MD is not used in case of DSI, reading will get some default value * set dpll_md = 0 */ pipe_config->dpll_hw_state.dpll_md = 0; - pclk = vlv_get_dsi_pclk(encoder, pipe_config->pipe_bpp); + if (IS_BROXTON(encoder->base.dev)) + pclk = bxt_get_dsi_pclk(encoder, pipe_config->pipe_bpp); + else if (IS_VALLEYVIEW(encoder->base.dev)) + pclk = vlv_get_dsi_pclk(encoder, pipe_config->pipe_bpp); + if (!pclk) return; @@ -646,6 +732,7 @@ intel_dsi_mode_valid(struct drm_connector *connector, { struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; + int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; DRM_DEBUG_KMS("\n"); @@ -659,6 +746,8 @@ intel_dsi_mode_valid(struct drm_connector *connector, return MODE_PANEL; if (mode->vdisplay > fixed_mode->vdisplay) return MODE_PANEL; + if (fixed_mode->clock > max_dotclk) + return MODE_CLOCK_HIGH; } return MODE_OK; @@ -687,7 +776,7 @@ static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count, } static void set_dsi_timings(struct drm_encoder *encoder, - const struct drm_display_mode *mode) + const struct drm_display_mode *adjusted_mode) { struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -699,10 +788,10 @@ static void set_dsi_timings(struct drm_encoder *encoder, u16 hactive, hfp, hsync, hbp, vfp, vsync, vbp; - hactive = mode->hdisplay; - hfp = mode->hsync_start - mode->hdisplay; - hsync = mode->hsync_end - mode->hsync_start; - hbp = mode->htotal - mode->hsync_end; + hactive = adjusted_mode->crtc_hdisplay; + hfp = adjusted_mode->crtc_hsync_start - adjusted_mode->crtc_hdisplay; + hsync = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; + hbp = adjusted_mode->crtc_htotal - adjusted_mode->crtc_hsync_end; if (intel_dsi->dual_link) { hactive /= 2; @@ -713,9 +802,9 @@ static void set_dsi_timings(struct drm_encoder *encoder, hbp /= 2; } - vfp = mode->vsync_start - mode->vdisplay; - vsync = mode->vsync_end - mode->vsync_start; - vbp = mode->vtotal - mode->vsync_end; + vfp = adjusted_mode->crtc_vsync_start - adjusted_mode->crtc_vdisplay; + vsync = adjusted_mode->crtc_vsync_end - adjusted_mode->crtc_vsync_start; + vbp = adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vsync_end; /* horizontal values are in terms of high speed byte clock */ hactive = txbyteclkhs(hactive, bpp, lane_count, @@ -726,6 +815,21 @@ static void set_dsi_timings(struct drm_encoder *encoder, hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio); for_each_dsi_port(port, intel_dsi->ports) { + if (IS_BROXTON(dev)) { + /* + * Program hdisplay and vdisplay on MIPI transcoder. + * This is different from calculated hactive and + * vactive, as they are calculated per channel basis, + * whereas these values should be based on resolution. + */ + I915_WRITE(BXT_MIPI_TRANS_HACTIVE(port), + adjusted_mode->crtc_hdisplay); + I915_WRITE(BXT_MIPI_TRANS_VACTIVE(port), + adjusted_mode->crtc_vdisplay); + I915_WRITE(BXT_MIPI_TRANS_VTOTAL(port), + adjusted_mode->crtc_vtotal); + } + I915_WRITE(MIPI_HACTIVE_AREA_COUNT(port), hactive); I915_WRITE(MIPI_HFP_COUNT(port), hfp); @@ -748,8 +852,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); - struct drm_display_mode *adjusted_mode = - &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; enum port port; unsigned int bpp = intel_crtc->config->pipe_bpp; u32 val, tmp; @@ -757,7 +860,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) DRM_DEBUG_KMS("pipe %c\n", pipe_name(intel_crtc->pipe)); - mode_hdisplay = adjusted_mode->hdisplay; + mode_hdisplay = adjusted_mode->crtc_hdisplay; if (intel_dsi->dual_link) { mode_hdisplay /= 2; @@ -766,16 +869,39 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) } for_each_dsi_port(port, intel_dsi->ports) { - /* escape clock divider, 20MHz, shared for A and C. - * device ready must be off when doing this! txclkesc? */ - tmp = I915_READ(MIPI_CTRL(PORT_A)); - tmp &= ~ESCAPE_CLOCK_DIVIDER_MASK; - I915_WRITE(MIPI_CTRL(PORT_A), tmp | ESCAPE_CLOCK_DIVIDER_1); - - /* read request priority is per pipe */ - tmp = I915_READ(MIPI_CTRL(port)); - tmp &= ~READ_REQUEST_PRIORITY_MASK; - I915_WRITE(MIPI_CTRL(port), tmp | READ_REQUEST_PRIORITY_HIGH); + if (IS_VALLEYVIEW(dev)) { + /* + * escape clock divider, 20MHz, shared for A and C. + * device ready must be off when doing this! txclkesc? + */ + tmp = I915_READ(MIPI_CTRL(PORT_A)); + tmp &= ~ESCAPE_CLOCK_DIVIDER_MASK; + I915_WRITE(MIPI_CTRL(PORT_A), tmp | + ESCAPE_CLOCK_DIVIDER_1); + + /* read request priority is per pipe */ + tmp = I915_READ(MIPI_CTRL(port)); + tmp &= ~READ_REQUEST_PRIORITY_MASK; + I915_WRITE(MIPI_CTRL(port), tmp | + READ_REQUEST_PRIORITY_HIGH); + } else if (IS_BROXTON(dev)) { + /* + * FIXME: + * BXT can connect any PIPE to any MIPI port. + * Select the pipe based on the MIPI port read from + * VBT for now. Pick PIPE A for MIPI port A and C + * for port C. + */ + tmp = I915_READ(MIPI_CTRL(port)); + tmp &= ~BXT_PIPE_SELECT_MASK; + + if (port == PORT_A) + tmp |= BXT_PIPE_SELECT_A; + else if (port == PORT_C) + tmp |= BXT_PIPE_SELECT_C; + + I915_WRITE(MIPI_CTRL(port), tmp); + } /* XXX: why here, why like this? handling in irq handler?! */ I915_WRITE(MIPI_INTR_STAT(port), 0xffffffff); @@ -784,7 +910,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) I915_WRITE(MIPI_DPHY_PARAM(port), intel_dsi->dphy_reg); I915_WRITE(MIPI_DPI_RESOLUTION(port), - adjusted_mode->vdisplay << VERTICAL_ADDRESS_SHIFT | + adjusted_mode->crtc_vdisplay << VERTICAL_ADDRESS_SHIFT | mode_hdisplay << HORIZONTAL_ADDRESS_SHIFT); } @@ -830,15 +956,15 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) if (is_vid_mode(intel_dsi) && intel_dsi->video_mode_format == VIDEO_MODE_BURST) { I915_WRITE(MIPI_HS_TX_TIMEOUT(port), - txbyteclkhs(adjusted_mode->htotal, bpp, - intel_dsi->lane_count, - intel_dsi->burst_mode_ratio) + 1); + txbyteclkhs(adjusted_mode->crtc_htotal, bpp, + intel_dsi->lane_count, + intel_dsi->burst_mode_ratio) + 1); } else { I915_WRITE(MIPI_HS_TX_TIMEOUT(port), - txbyteclkhs(adjusted_mode->vtotal * - adjusted_mode->htotal, - bpp, intel_dsi->lane_count, - intel_dsi->burst_mode_ratio) + 1); + txbyteclkhs(adjusted_mode->crtc_vtotal * + adjusted_mode->crtc_htotal, + bpp, intel_dsi->lane_count, + intel_dsi->burst_mode_ratio) + 1); } I915_WRITE(MIPI_LP_RX_TIMEOUT(port), intel_dsi->lp_rx_timeout); I915_WRITE(MIPI_TURN_AROUND_TIMEOUT(port), @@ -852,6 +978,17 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) I915_WRITE(MIPI_INIT_COUNT(port), txclkesc(intel_dsi->escape_clk_div, 100)); + if (IS_BROXTON(dev) && (!intel_dsi->dual_link)) { + /* + * BXT spec says write MIPI_INIT_COUNT for + * both the ports, even if only one is + * getting used. So write the other port + * if not in dual link mode. + */ + I915_WRITE(MIPI_INIT_COUNT(port == + PORT_A ? PORT_C : PORT_A), + intel_dsi->init_count); + } /* recovery disables */ I915_WRITE(MIPI_EOT_DISABLE(port), tmp); @@ -898,15 +1035,6 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) } } -static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder) -{ - DRM_DEBUG_KMS("\n"); - - intel_dsi_prepare(encoder); - - vlv_enable_dsi_pll(encoder); -} - static enum drm_connector_status intel_dsi_detect(struct drm_connector *connector, bool force) { @@ -955,6 +1083,11 @@ static void intel_dsi_encoder_destroy(struct drm_encoder *encoder) /* XXX: Logically this call belongs in the panel driver. */ drm_panel_remove(intel_dsi->panel); } + + /* dispose of the gpios */ + if (intel_dsi->gpio_panel) + gpiod_put(intel_dsi->gpio_panel); + intel_encoder_destroy(encoder); } @@ -969,7 +1102,7 @@ static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs }; static const struct drm_connector_funcs intel_dsi_connector_funcs = { - .dpms = intel_connector_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_dsi_detect, .destroy = intel_dsi_connector_destroy, .fill_modes = drm_helper_probe_single_connector_modes, @@ -1019,12 +1152,10 @@ void intel_dsi_init(struct drm_device *dev) connector = &intel_connector->base; - drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI); + drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI, + NULL); - /* XXX: very likely not all of these are needed */ - intel_encoder->hot_plug = intel_dsi_hot_plug; intel_encoder->compute_config = intel_dsi_compute_config; - intel_encoder->pre_pll_enable = intel_dsi_pre_pll_enable; intel_encoder->pre_enable = intel_dsi_pre_enable; intel_encoder->enable = intel_dsi_enable_nop; intel_encoder->disable = intel_dsi_pre_disable; @@ -1036,11 +1167,7 @@ void intel_dsi_init(struct drm_device *dev) intel_connector->unregister = intel_connector_unregister; /* Pipe A maps to MIPI DSI port A, pipe B maps to MIPI DSI port C */ - if (dev_priv->vbt.dsi.config->dual_link) { - /* XXX: does dual link work on either pipe? */ - intel_encoder->crtc_mask = (1 << PIPE_A); - intel_dsi->ports = ((1 << PORT_A) | (1 << PORT_C)); - } else if (dev_priv->vbt.dsi.port == DVO_PORT_MIPIA) { + if (dev_priv->vbt.dsi.port == DVO_PORT_MIPIA) { intel_encoder->crtc_mask = (1 << PIPE_A); intel_dsi->ports = (1 << PORT_A); } else if (dev_priv->vbt.dsi.port == DVO_PORT_MIPIC) { @@ -1048,6 +1175,9 @@ void intel_dsi_init(struct drm_device *dev) intel_dsi->ports = (1 << PORT_C); } + if (dev_priv->vbt.dsi.config->dual_link) + intel_dsi->ports = ((1 << PORT_A) | (1 << PORT_C)); + /* Create a DSI host (and a device) for each port. */ for_each_dsi_port(port, intel_dsi->ports) { struct intel_dsi_host *host; @@ -1071,6 +1201,20 @@ void intel_dsi_init(struct drm_device *dev) goto err; } + /* + * In case of BYT with CRC PMIC, we need to use GPIO for + * Panel control. + */ + if (dev_priv->vbt.dsi.config->pwm_blc == PPS_BLC_PMIC) { + intel_dsi->gpio_panel = + gpiod_get(dev->dev, "panel", GPIOD_OUT_HIGH); + + if (IS_ERR(intel_dsi->gpio_panel)) { + DRM_ERROR("Failed to own gpio for panel control\n"); + intel_dsi->gpio_panel = NULL; + } + } + intel_encoder->type = INTEL_OUTPUT_DSI; intel_encoder->cloneable = 0; drm_connector_init(dev, connector, &intel_dsi_connector_funcs, @@ -1104,6 +1248,7 @@ void intel_dsi_init(struct drm_device *dev) } intel_panel_init(&intel_connector->panel, fixed_mode, NULL); + intel_panel_setup_backlight(connector, INVALID_PIPE); return; diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h index 2784ac442368..02551ff228c2 100644 --- a/drivers/gpu/drm/i915/intel_dsi.h +++ b/drivers/gpu/drm/i915/intel_dsi.h @@ -42,6 +42,9 @@ struct intel_dsi { struct drm_panel *panel; struct intel_dsi_host *dsi_hosts[I915_MAX_PORTS]; + /* GPIO Desc for CRC based Panel control */ + struct gpio_desc *gpio_panel; + struct intel_connector *attached_connector; /* bit mask of ports being driven */ @@ -114,16 +117,19 @@ static inline struct intel_dsi_host *to_intel_dsi_host(struct mipi_dsi_host *h) #define for_each_dsi_port(__port, __ports_mask) \ for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++) \ - if ((__ports_mask) & (1 << (__port))) + for_each_if ((__ports_mask) & (1 << (__port))) static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder) { return container_of(encoder, struct intel_dsi, base.base); } -extern void vlv_enable_dsi_pll(struct intel_encoder *encoder); -extern void vlv_disable_dsi_pll(struct intel_encoder *encoder); +extern void intel_enable_dsi_pll(struct intel_encoder *encoder); +extern void intel_disable_dsi_pll(struct intel_encoder *encoder); extern u32 vlv_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp); +extern u32 bxt_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp); +extern void intel_dsi_reset_clocks(struct intel_encoder *encoder, + enum port port); struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id); diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c index d20cf37b6901..cb3cf3986212 100644 --- a/drivers/gpu/drm/i915/intel_dsi_pll.c +++ b/drivers/gpu/drm/i915/intel_dsi_pll.c @@ -38,6 +38,27 @@ #define DSI_HFP_PACKET_EXTRA_SIZE 6 #define DSI_EOTP_PACKET_SIZE 4 +static int dsi_pixel_format_bpp(int pixel_format) +{ + int bpp; + + switch (pixel_format) { + default: + case VID_MODE_FORMAT_RGB888: + case VID_MODE_FORMAT_RGB666_LOOSE: + bpp = 24; + break; + case VID_MODE_FORMAT_RGB666: + bpp = 18; + break; + case VID_MODE_FORMAT_RGB565: + bpp = 16; + break; + } + + return bpp; +} + struct dsi_mnp { u32 dsi_pll_ctrl; u32 dsi_pll_div; @@ -46,8 +67,8 @@ struct dsi_mnp { static const u32 lfsr_converts[] = { 426, 469, 234, 373, 442, 221, 110, 311, 411, /* 62 - 70 */ 461, 486, 243, 377, 188, 350, 175, 343, 427, 213, /* 71 - 80 */ - 106, 53, 282, 397, 354, 227, 113, 56, 284, 142, /* 81 - 90 */ - 71, 35 /* 91 - 92 */ + 106, 53, 282, 397, 454, 227, 113, 56, 284, 142, /* 81 - 90 */ + 71, 35, 273, 136, 324, 418, 465, 488, 500, 506 /* 91 - 100 */ }; #ifdef DSI_CLK_FROM_RR @@ -65,19 +86,7 @@ static u32 dsi_rr_formula(const struct drm_display_mode *mode, u32 dsi_bit_clock_hz; u32 dsi_clk; - switch (pixel_format) { - default: - case VID_MODE_FORMAT_RGB888: - case VID_MODE_FORMAT_RGB666_LOOSE: - bpp = 24; - break; - case VID_MODE_FORMAT_RGB666: - bpp = 18; - break; - case VID_MODE_FORMAT_RGB565: - bpp = 16; - break; - } + bpp = dsi_pixel_format_bpp(pixel_format); hactive = mode->hdisplay; vactive = mode->vdisplay; @@ -137,21 +146,7 @@ static u32 dsi_rr_formula(const struct drm_display_mode *mode, static u32 dsi_clk_from_pclk(u32 pclk, int pixel_format, int lane_count) { u32 dsi_clk_khz; - u32 bpp; - - switch (pixel_format) { - default: - case VID_MODE_FORMAT_RGB888: - case VID_MODE_FORMAT_RGB666_LOOSE: - bpp = 24; - break; - case VID_MODE_FORMAT_RGB666: - bpp = 18; - break; - case VID_MODE_FORMAT_RGB565: - bpp = 16; - break; - } + u32 bpp = dsi_pixel_format_bpp(pixel_format); /* DSI data rate = pixel clock * bits per pixel / lane count pixel clock is converted from KHz to Hz */ @@ -162,11 +157,13 @@ static u32 dsi_clk_from_pclk(u32 pclk, int pixel_format, int lane_count) #endif -static int dsi_calc_mnp(int target_dsi_clk, struct dsi_mnp *dsi_mnp) +static int dsi_calc_mnp(struct drm_i915_private *dev_priv, + struct dsi_mnp *dsi_mnp, int target_dsi_clk) { unsigned int calc_m = 0, calc_p = 0; - unsigned int m, n = 1, p; - int ref_clk = 25000; + unsigned int m_min, m_max, p_min = 2, p_max = 6; + unsigned int m, n, p; + int ref_clk; int delta = target_dsi_clk; u32 m_seed; @@ -176,8 +173,20 @@ static int dsi_calc_mnp(int target_dsi_clk, struct dsi_mnp *dsi_mnp) return -ECHRNG; } - for (m = 62; m <= 92 && delta; m++) { - for (p = 2; p <= 6 && delta; p++) { + if (IS_CHERRYVIEW(dev_priv)) { + ref_clk = 100000; + n = 4; + m_min = 70; + m_max = 96; + } else { + ref_clk = 25000; + n = 1; + m_min = 62; + m_max = 92; + } + + for (m = m_min; m <= m_max && delta; m++) { + for (p = p_min; p <= p_max && delta; p++) { /* * Find the optimal m and p divisors with minimal delta * +/- the required clock @@ -217,7 +226,7 @@ static void vlv_configure_dsi_pll(struct intel_encoder *encoder) dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format, intel_dsi->lane_count); - ret = dsi_calc_mnp(dsi_clk, &dsi_mnp); + ret = dsi_calc_mnp(dev_priv, &dsi_mnp, dsi_clk); if (ret) { DRM_DEBUG_KMS("dsi_calc_mnp failed\n"); return; @@ -237,7 +246,7 @@ static void vlv_configure_dsi_pll(struct intel_encoder *encoder) vlv_cck_write(dev_priv, CCK_REG_DSI_PLL_CONTROL, dsi_mnp.dsi_pll_ctrl); } -void vlv_enable_dsi_pll(struct intel_encoder *encoder) +static void vlv_enable_dsi_pll(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; u32 tmp; @@ -267,7 +276,7 @@ void vlv_enable_dsi_pll(struct intel_encoder *encoder) DRM_DEBUG_KMS("DSI PLL locked\n"); } -void vlv_disable_dsi_pll(struct intel_encoder *encoder) +static void vlv_disable_dsi_pll(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; u32 tmp; @@ -284,23 +293,29 @@ void vlv_disable_dsi_pll(struct intel_encoder *encoder) mutex_unlock(&dev_priv->sb_lock); } -static void assert_bpp_mismatch(int pixel_format, int pipe_bpp) +static void bxt_disable_dsi_pll(struct intel_encoder *encoder) { - int bpp; + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + u32 val; - switch (pixel_format) { - default: - case VID_MODE_FORMAT_RGB888: - case VID_MODE_FORMAT_RGB666_LOOSE: - bpp = 24; - break; - case VID_MODE_FORMAT_RGB666: - bpp = 18; - break; - case VID_MODE_FORMAT_RGB565: - bpp = 16; - break; - } + DRM_DEBUG_KMS("\n"); + + val = I915_READ(BXT_DSI_PLL_ENABLE); + val &= ~BXT_DSI_PLL_DO_ENABLE; + I915_WRITE(BXT_DSI_PLL_ENABLE, val); + + /* + * PLL lock should deassert within 200us. + * Wait up to 1ms before timing out. + */ + if (wait_for((I915_READ(BXT_DSI_PLL_ENABLE) + & BXT_DSI_PLL_LOCKED) == 0, 1)) + DRM_ERROR("Timeout waiting for PLL lock deassertion\n"); +} + +static void assert_bpp_mismatch(int pixel_format, int pipe_bpp) +{ + int bpp = dsi_pixel_format_bpp(pixel_format); WARN(bpp != pipe_bpp, "bpp match assertion failure (expected %d, current %d)\n", @@ -368,3 +383,222 @@ u32 vlv_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp) return pclk; } + +u32 bxt_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp) +{ + u32 pclk; + u32 dsi_clk; + u32 dsi_ratio; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + + /* Divide by zero */ + if (!pipe_bpp) { + DRM_ERROR("Invalid BPP(0)\n"); + return 0; + } + + dsi_ratio = I915_READ(BXT_DSI_PLL_CTL) & + BXT_DSI_PLL_RATIO_MASK; + + /* Invalid DSI ratio ? */ + if (dsi_ratio < BXT_DSI_PLL_RATIO_MIN || + dsi_ratio > BXT_DSI_PLL_RATIO_MAX) { + DRM_ERROR("Invalid DSI pll ratio(%u) programmed\n", dsi_ratio); + return 0; + } + + dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2; + + /* pixel_format and pipe_bpp should agree */ + assert_bpp_mismatch(intel_dsi->pixel_format, pipe_bpp); + + pclk = DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, pipe_bpp); + + DRM_DEBUG_DRIVER("Calculated pclk=%u\n", pclk); + return pclk; +} + +static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) +{ + u32 temp; + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + + temp = I915_READ(MIPI_CTRL(port)); + temp &= ~ESCAPE_CLOCK_DIVIDER_MASK; + I915_WRITE(MIPI_CTRL(port), temp | + intel_dsi->escape_clk_div << + ESCAPE_CLOCK_DIVIDER_SHIFT); +} + +/* Program BXT Mipi clocks and dividers */ +static void bxt_dsi_program_clocks(struct drm_device *dev, enum port port) +{ + u32 tmp; + u32 divider; + u32 dsi_rate; + u32 pll_ratio; + struct drm_i915_private *dev_priv = dev->dev_private; + + /* Clear old configurations */ + tmp = I915_READ(BXT_MIPI_CLOCK_CTL); + tmp &= ~(BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port)); + tmp &= ~(BXT_MIPI_RX_ESCLK_FIXDIV_MASK(port)); + tmp &= ~(BXT_MIPI_ESCLK_VAR_DIV_MASK(port)); + tmp &= ~(BXT_MIPI_DPHY_DIVIDER_MASK(port)); + + /* Get the current DSI rate(actual) */ + pll_ratio = I915_READ(BXT_DSI_PLL_CTL) & + BXT_DSI_PLL_RATIO_MASK; + dsi_rate = (BXT_REF_CLOCK_KHZ * pll_ratio) / 2; + + /* Max possible output of clock is 39.5 MHz, program value -1 */ + divider = (dsi_rate / BXT_MAX_VAR_OUTPUT_KHZ) - 1; + tmp |= BXT_MIPI_ESCLK_VAR_DIV(port, divider); + + /* + * Tx escape clock must be as close to 20MHz possible, but should + * not exceed it. Hence select divide by 2 + */ + tmp |= BXT_MIPI_TX_ESCLK_8XDIV_BY2(port); + + tmp |= BXT_MIPI_RX_ESCLK_8X_BY3(port); + + I915_WRITE(BXT_MIPI_CLOCK_CTL, tmp); +} + +static bool bxt_configure_dsi_pll(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u8 dsi_ratio; + u32 dsi_clk; + u32 val; + + dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format, + intel_dsi->lane_count); + + /* + * From clock diagram, to get PLL ratio divider, divide double of DSI + * link rate (i.e., 2*8x=16x frequency value) by ref clock. Make sure to + * round 'up' the result + */ + dsi_ratio = DIV_ROUND_UP(dsi_clk * 2, BXT_REF_CLOCK_KHZ); + if (dsi_ratio < BXT_DSI_PLL_RATIO_MIN || + dsi_ratio > BXT_DSI_PLL_RATIO_MAX) { + DRM_ERROR("Cant get a suitable ratio from DSI PLL ratios\n"); + return false; + } + + /* + * Program DSI ratio and Select MIPIC and MIPIA PLL output as 8x + * Spec says both have to be programmed, even if one is not getting + * used. Configure MIPI_CLOCK_CTL dividers in modeset + */ + val = I915_READ(BXT_DSI_PLL_CTL); + val &= ~BXT_DSI_PLL_PVD_RATIO_MASK; + val &= ~BXT_DSI_FREQ_SEL_MASK; + val &= ~BXT_DSI_PLL_RATIO_MASK; + val |= (dsi_ratio | BXT_DSIA_16X_BY2 | BXT_DSIC_16X_BY2); + + /* As per recommendation from hardware team, + * Prog PVD ratio =1 if dsi ratio <= 50 + */ + if (dsi_ratio <= 50) { + val &= ~BXT_DSI_PLL_PVD_RATIO_MASK; + val |= BXT_DSI_PLL_PVD_RATIO_1; + } + + I915_WRITE(BXT_DSI_PLL_CTL, val); + POSTING_READ(BXT_DSI_PLL_CTL); + + return true; +} + +static void bxt_enable_dsi_pll(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + enum port port; + u32 val; + + DRM_DEBUG_KMS("\n"); + + val = I915_READ(BXT_DSI_PLL_ENABLE); + + if (val & BXT_DSI_PLL_DO_ENABLE) { + WARN(1, "DSI PLL already enabled. Disabling it.\n"); + val &= ~BXT_DSI_PLL_DO_ENABLE; + I915_WRITE(BXT_DSI_PLL_ENABLE, val); + } + + /* Configure PLL vales */ + if (!bxt_configure_dsi_pll(encoder)) { + DRM_ERROR("Configure DSI PLL failed, abort PLL enable\n"); + return; + } + + /* Program TX, RX, Dphy clocks */ + for_each_dsi_port(port, intel_dsi->ports) + bxt_dsi_program_clocks(encoder->base.dev, port); + + /* Enable DSI PLL */ + val = I915_READ(BXT_DSI_PLL_ENABLE); + val |= BXT_DSI_PLL_DO_ENABLE; + I915_WRITE(BXT_DSI_PLL_ENABLE, val); + + /* Timeout and fail if PLL not locked */ + if (wait_for(I915_READ(BXT_DSI_PLL_ENABLE) & BXT_DSI_PLL_LOCKED, 1)) { + DRM_ERROR("Timed out waiting for DSI PLL to lock\n"); + return; + } + + DRM_DEBUG_KMS("DSI PLL locked\n"); +} + +void intel_enable_dsi_pll(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + + if (IS_VALLEYVIEW(dev)) + vlv_enable_dsi_pll(encoder); + else if (IS_BROXTON(dev)) + bxt_enable_dsi_pll(encoder); +} + +void intel_disable_dsi_pll(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + + if (IS_VALLEYVIEW(dev)) + vlv_disable_dsi_pll(encoder); + else if (IS_BROXTON(dev)) + bxt_disable_dsi_pll(encoder); +} + +static void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) +{ + u32 tmp; + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + /* Clear old configurations */ + tmp = I915_READ(BXT_MIPI_CLOCK_CTL); + tmp &= ~(BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port)); + tmp &= ~(BXT_MIPI_RX_ESCLK_FIXDIV_MASK(port)); + tmp &= ~(BXT_MIPI_ESCLK_VAR_DIV_MASK(port)); + tmp &= ~(BXT_MIPI_DPHY_DIVIDER_MASK(port)); + I915_WRITE(BXT_MIPI_CLOCK_CTL, tmp); + I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP); +} + +void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) +{ + struct drm_device *dev = encoder->base.dev; + + if (IS_BROXTON(dev)) + bxt_dsi_reset_clocks(encoder, port); + else if (IS_VALLEYVIEW(dev)) + vlv_dsi_reset_clocks(encoder, port); +} diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index ece5bd754f85..286baec979c8 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -44,6 +44,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_TMDS, .name = "sil164", .dvo_reg = DVOC, + .dvo_srcdim_reg = DVOC_SRCDIM, .slave_addr = SIL164_ADDR, .dev_ops = &sil164_ops, }, @@ -51,6 +52,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_TMDS, .name = "ch7xxx", .dvo_reg = DVOC, + .dvo_srcdim_reg = DVOC_SRCDIM, .slave_addr = CH7xxx_ADDR, .dev_ops = &ch7xxx_ops, }, @@ -58,6 +60,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_TMDS, .name = "ch7xxx", .dvo_reg = DVOC, + .dvo_srcdim_reg = DVOC_SRCDIM, .slave_addr = 0x75, /* For some ch7010 */ .dev_ops = &ch7xxx_ops, }, @@ -65,6 +68,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_LVDS, .name = "ivch", .dvo_reg = DVOA, + .dvo_srcdim_reg = DVOA_SRCDIM, .slave_addr = 0x02, /* Might also be 0x44, 0x84, 0xc4 */ .dev_ops = &ivch_ops, }, @@ -72,6 +76,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_TMDS, .name = "tfp410", .dvo_reg = DVOC, + .dvo_srcdim_reg = DVOC_SRCDIM, .slave_addr = TFP410_ADDR, .dev_ops = &tfp410_ops, }, @@ -79,6 +84,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_LVDS, .name = "ch7017", .dvo_reg = DVOC, + .dvo_srcdim_reg = DVOC_SRCDIM, .slave_addr = 0x75, .gpio = GMBUS_PIN_DPB, .dev_ops = &ch7017_ops, @@ -87,6 +93,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = { .type = INTEL_DVO_CHIP_TMDS, .name = "ns2501", .dvo_reg = DVOB, + .dvo_srcdim_reg = DVOB_SRCDIM, .slave_addr = NS2501_ADDR, .dev_ops = &ns2501_ops, } @@ -97,7 +104,8 @@ struct intel_dvo { struct intel_dvo_device dev; - struct drm_display_mode *panel_fixed_mode; + struct intel_connector *attached_connector; + bool panel_wants_dither; }; @@ -170,7 +178,7 @@ static void intel_disable_dvo(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; struct intel_dvo *intel_dvo = enc_to_dvo(encoder); - u32 dvo_reg = intel_dvo->dev.dvo_reg; + i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg; u32 temp = I915_READ(dvo_reg); intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false); @@ -183,7 +191,7 @@ static void intel_enable_dvo(struct intel_encoder *encoder) struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; struct intel_dvo *intel_dvo = enc_to_dvo(encoder); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - u32 dvo_reg = intel_dvo->dev.dvo_reg; + i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg; u32 temp = I915_READ(dvo_reg); intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev, @@ -196,68 +204,33 @@ static void intel_enable_dvo(struct intel_encoder *encoder) intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true); } -/* Special dpms function to support cloning between dvo/sdvo/crt. */ -static void intel_dvo_dpms(struct drm_connector *connector, int mode) -{ - struct intel_dvo *intel_dvo = intel_attached_dvo(connector); - struct drm_crtc *crtc; - struct intel_crtc_state *config; - - /* dvo supports only 2 dpms states. */ - if (mode != DRM_MODE_DPMS_ON) - mode = DRM_MODE_DPMS_OFF; - - if (mode == connector->dpms) - return; - - connector->dpms = mode; - - /* Only need to change hw state when actually enabled */ - crtc = intel_dvo->base.base.crtc; - if (!crtc) { - intel_dvo->base.connectors_active = false; - return; - } - - /* We call connector dpms manually below in case pipe dpms doesn't - * change due to cloning. */ - if (mode == DRM_MODE_DPMS_ON) { - config = to_intel_crtc(crtc)->config; - - intel_dvo->base.connectors_active = true; - - intel_crtc_update_dpms(crtc); - - intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true); - } else { - intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false); - - intel_dvo->base.connectors_active = false; - - intel_crtc_update_dpms(crtc); - } - - intel_modeset_check_state(connector->dev); -} - static enum drm_mode_status intel_dvo_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { struct intel_dvo *intel_dvo = intel_attached_dvo(connector); + const struct drm_display_mode *fixed_mode = + to_intel_connector(connector)->panel.fixed_mode; + int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; + int target_clock = mode->clock; if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; /* XXX: Validate clock range */ - if (intel_dvo->panel_fixed_mode) { - if (mode->hdisplay > intel_dvo->panel_fixed_mode->hdisplay) + if (fixed_mode) { + if (mode->hdisplay > fixed_mode->hdisplay) return MODE_PANEL; - if (mode->vdisplay > intel_dvo->panel_fixed_mode->vdisplay) + if (mode->vdisplay > fixed_mode->vdisplay) return MODE_PANEL; + + target_clock = fixed_mode->clock; } + if (target_clock > max_dotclk) + return MODE_CLOCK_HIGH; + return intel_dvo->dev.dev_ops->mode_valid(&intel_dvo->dev, mode); } @@ -265,6 +238,8 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct intel_dvo *intel_dvo = enc_to_dvo(encoder); + const struct drm_display_mode *fixed_mode = + intel_dvo->attached_connector->panel.fixed_mode; struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; /* If we have timings from the BIOS for the panel, put them in @@ -272,21 +247,8 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder, * with the panel scaling set up to source from the H/VDisplay * of the original mode. */ - if (intel_dvo->panel_fixed_mode != NULL) { -#define C(x) adjusted_mode->x = intel_dvo->panel_fixed_mode->x - C(hdisplay); - C(hsync_start); - C(hsync_end); - C(htotal); - C(vdisplay); - C(vsync_start); - C(vsync_end); - C(vtotal); - C(clock); -#undef C - - drm_mode_set_crtcinfo(adjusted_mode, 0); - } + if (fixed_mode) + intel_fixed_panel_mode(fixed_mode, adjusted_mode); return true; } @@ -296,24 +258,12 @@ static void intel_dvo_pre_enable(struct intel_encoder *encoder) struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; struct intel_dvo *intel_dvo = enc_to_dvo(encoder); int pipe = crtc->pipe; u32 dvo_val; - u32 dvo_reg = intel_dvo->dev.dvo_reg, dvo_srcdim_reg; - - switch (dvo_reg) { - case DVOA: - default: - dvo_srcdim_reg = DVOA_SRCDIM; - break; - case DVOB: - dvo_srcdim_reg = DVOB_SRCDIM; - break; - case DVOC: - dvo_srcdim_reg = DVOC_SRCDIM; - break; - } + i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg; + i915_reg_t dvo_srcdim_reg = intel_dvo->dev.dvo_srcdim_reg; /* Save the data order, since I don't know what it should be set to. */ dvo_val = I915_READ(dvo_reg) & @@ -330,11 +280,11 @@ static void intel_dvo_pre_enable(struct intel_encoder *encoder) dvo_val |= DVO_VSYNC_ACTIVE_HIGH; /*I915_WRITE(DVOB_SRCDIM, - (adjusted_mode->hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) | - (adjusted_mode->VDisplay << DVO_SRCDIM_VERTICAL_SHIFT));*/ + (adjusted_mode->crtc_hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) | + (adjusted_mode->crtc_vdisplay << DVO_SRCDIM_VERTICAL_SHIFT));*/ I915_WRITE(dvo_srcdim_reg, - (adjusted_mode->hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) | - (adjusted_mode->vdisplay << DVO_SRCDIM_VERTICAL_SHIFT)); + (adjusted_mode->crtc_hdisplay << DVO_SRCDIM_HORIZONTAL_SHIFT) | + (adjusted_mode->crtc_vdisplay << DVO_SRCDIM_VERTICAL_SHIFT)); /*I915_WRITE(DVOB, dvo_val);*/ I915_WRITE(dvo_reg, dvo_val); } @@ -355,8 +305,9 @@ intel_dvo_detect(struct drm_connector *connector, bool force) static int intel_dvo_get_modes(struct drm_connector *connector) { - struct intel_dvo *intel_dvo = intel_attached_dvo(connector); struct drm_i915_private *dev_priv = connector->dev->dev_private; + const struct drm_display_mode *fixed_mode = + to_intel_connector(connector)->panel.fixed_mode; /* We should probably have an i2c driver get_modes function for those * devices which will have a fixed set of modes determined by the chip @@ -368,9 +319,9 @@ static int intel_dvo_get_modes(struct drm_connector *connector) if (!list_empty(&connector->probed_modes)) return 1; - if (intel_dvo->panel_fixed_mode != NULL) { + if (fixed_mode) { struct drm_display_mode *mode; - mode = drm_mode_duplicate(connector->dev, intel_dvo->panel_fixed_mode); + mode = drm_mode_duplicate(connector->dev, fixed_mode); if (mode) { drm_mode_probed_add(connector, mode); return 1; @@ -383,11 +334,12 @@ static int intel_dvo_get_modes(struct drm_connector *connector) static void intel_dvo_destroy(struct drm_connector *connector) { drm_connector_cleanup(connector); + intel_panel_fini(&to_intel_connector(connector)->panel); kfree(connector); } static const struct drm_connector_funcs intel_dvo_connector_funcs = { - .dpms = intel_dvo_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_dvo_detect, .destroy = intel_dvo_destroy, .fill_modes = drm_helper_probe_single_connector_modes, @@ -409,8 +361,6 @@ static void intel_dvo_enc_destroy(struct drm_encoder *encoder) if (intel_dvo->dev.dev_ops->destroy) intel_dvo->dev.dev_ops->destroy(&intel_dvo->dev); - kfree(intel_dvo->panel_fixed_mode); - intel_encoder_destroy(encoder); } @@ -475,9 +425,11 @@ void intel_dvo_init(struct drm_device *dev) return; } + intel_dvo->attached_connector = intel_connector; + intel_encoder = &intel_dvo->base; drm_encoder_init(dev, &intel_encoder->base, - &intel_dvo_enc_funcs, encoder_type); + &intel_dvo_enc_funcs, encoder_type, NULL); intel_encoder->disable = intel_disable_dvo; intel_encoder->enable = intel_enable_dvo; @@ -579,8 +531,9 @@ void intel_dvo_init(struct drm_device *dev) * headers, likely), so for now, just get the current * mode being output through DVO. */ - intel_dvo->panel_fixed_mode = - intel_dvo_get_current_mode(connector); + intel_panel_init(&intel_connector->panel, + intel_dvo_get_current_mode(connector), + NULL); intel_dvo->panel_wants_dither = true; } diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c index 6abb83432d4d..a1988a486b92 100644 --- a/drivers/gpu/drm/i915/intel_fbc.c +++ b/drivers/gpu/drm/i915/intel_fbc.c @@ -41,12 +41,79 @@ #include "intel_drv.h" #include "i915_drv.h" -static void i8xx_fbc_disable(struct drm_device *dev) +static inline bool fbc_supported(struct drm_i915_private *dev_priv) +{ + return dev_priv->fbc.activate != NULL; +} + +static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv) +{ + return IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8; +} + +static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv) +{ + return INTEL_INFO(dev_priv)->gen < 4; +} + +/* + * In some platforms where the CRTC's x:0/y:0 coordinates doesn't match the + * frontbuffer's x:0/y:0 coordinates we lie to the hardware about the plane's + * origin so the x and y offsets can actually fit the registers. As a + * consequence, the fence doesn't really start exactly at the display plane + * address we program because it starts at the real start of the buffer, so we + * have to take this into consideration here. + */ +static unsigned int get_crtc_fence_y_offset(struct intel_crtc *crtc) +{ + return crtc->base.y - crtc->adjusted_y; +} + +/* + * For SKL+, the plane source size used by the hardware is based on the value we + * write to the PLANE_SIZE register. For BDW-, the hardware looks at the value + * we wrote to PIPESRC. + */ +static void intel_fbc_get_plane_source_size(struct intel_crtc *crtc, + int *width, int *height) +{ + struct intel_plane_state *plane_state = + to_intel_plane_state(crtc->base.primary->state); + int w, h; + + if (intel_rotation_90_or_270(plane_state->base.rotation)) { + w = drm_rect_height(&plane_state->src) >> 16; + h = drm_rect_width(&plane_state->src) >> 16; + } else { + w = drm_rect_width(&plane_state->src) >> 16; + h = drm_rect_height(&plane_state->src) >> 16; + } + + if (width) + *width = w; + if (height) + *height = h; +} + +static int intel_fbc_calculate_cfb_size(struct intel_crtc *crtc, + struct drm_framebuffer *fb) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + int lines; + + intel_fbc_get_plane_source_size(crtc, NULL, &lines); + if (INTEL_INFO(dev_priv)->gen >= 7) + lines = min(lines, 2048); + + /* Hardware needs the full buffer stride, not just the active area. */ + return lines * fb->pitches[0]; +} + +static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 fbc_ctl; - dev_priv->fbc.enabled = false; + dev_priv->fbc.active = false; /* Disable compression */ fbc_ctl = I915_READ(FBC_CONTROL); @@ -61,141 +128,124 @@ static void i8xx_fbc_disable(struct drm_device *dev) DRM_DEBUG_KMS("FBC idle timed out\n"); return; } - - DRM_DEBUG_KMS("disabled FBC\n"); } -static void i8xx_fbc_enable(struct drm_crtc *crtc) +static void i8xx_fbc_activate(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->primary->fb; + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_framebuffer *fb = crtc->base.primary->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int cfb_pitch; int i; u32 fbc_ctl; - dev_priv->fbc.enabled = true; + dev_priv->fbc.active = true; /* Note: fbc.threshold == 1 for i8xx */ - cfb_pitch = dev_priv->fbc.uncompressed_size / FBC_LL_SIZE; + cfb_pitch = intel_fbc_calculate_cfb_size(crtc, fb) / FBC_LL_SIZE; if (fb->pitches[0] < cfb_pitch) cfb_pitch = fb->pitches[0]; /* FBC_CTL wants 32B or 64B units */ - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) cfb_pitch = (cfb_pitch / 32) - 1; else cfb_pitch = (cfb_pitch / 64) - 1; /* Clear old tags */ for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++) - I915_WRITE(FBC_TAG + (i * 4), 0); + I915_WRITE(FBC_TAG(i), 0); - if (IS_GEN4(dev)) { + if (IS_GEN4(dev_priv)) { u32 fbc_ctl2; /* Set it up... */ fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE; - fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane); + fbc_ctl2 |= FBC_CTL_PLANE(crtc->plane); I915_WRITE(FBC_CONTROL2, fbc_ctl2); - I915_WRITE(FBC_FENCE_OFF, crtc->y); + I915_WRITE(FBC_FENCE_OFF, get_crtc_fence_y_offset(crtc)); } /* enable it... */ fbc_ctl = I915_READ(FBC_CONTROL); fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT; fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC; - if (IS_I945GM(dev)) + if (IS_I945GM(dev_priv)) fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */ fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT; fbc_ctl |= obj->fence_reg; I915_WRITE(FBC_CONTROL, fbc_ctl); - - DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n", - cfb_pitch, crtc->y, plane_name(intel_crtc->plane)); } -static bool i8xx_fbc_enabled(struct drm_device *dev) +static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - return I915_READ(FBC_CONTROL) & FBC_CTL_EN; } -static void g4x_fbc_enable(struct drm_crtc *crtc) +static void g4x_fbc_activate(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->primary->fb; + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_framebuffer *fb = crtc->base.primary->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 dpfc_ctl; - dev_priv->fbc.enabled = true; + dev_priv->fbc.active = true; - dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN; + dpfc_ctl = DPFC_CTL_PLANE(crtc->plane) | DPFC_SR_EN; if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) dpfc_ctl |= DPFC_CTL_LIMIT_2X; else dpfc_ctl |= DPFC_CTL_LIMIT_1X; dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg; - I915_WRITE(DPFC_FENCE_YOFF, crtc->y); + I915_WRITE(DPFC_FENCE_YOFF, get_crtc_fence_y_offset(crtc)); /* enable it... */ I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); - - DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); } -static void g4x_fbc_disable(struct drm_device *dev) +static void g4x_fbc_deactivate(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 dpfc_ctl; - dev_priv->fbc.enabled = false; + dev_priv->fbc.active = false; /* Disable compression */ dpfc_ctl = I915_READ(DPFC_CONTROL); if (dpfc_ctl & DPFC_CTL_EN) { dpfc_ctl &= ~DPFC_CTL_EN; I915_WRITE(DPFC_CONTROL, dpfc_ctl); - - DRM_DEBUG_KMS("disabled FBC\n"); } } -static bool g4x_fbc_enabled(struct drm_device *dev) +static bool g4x_fbc_is_active(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN; } -static void intel_fbc_nuke(struct drm_i915_private *dev_priv) +/* This function forces a CFB recompression through the nuke operation. */ +static void intel_fbc_recompress(struct drm_i915_private *dev_priv) { I915_WRITE(MSG_FBC_REND_STATE, FBC_REND_NUKE); POSTING_READ(MSG_FBC_REND_STATE); } -static void ilk_fbc_enable(struct drm_crtc *crtc) +static void ilk_fbc_activate(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->primary->fb; + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_framebuffer *fb = crtc->base.primary->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 dpfc_ctl; + int threshold = dev_priv->fbc.threshold; + unsigned int y_offset; - dev_priv->fbc.enabled = true; + dev_priv->fbc.active = true; - dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane); + dpfc_ctl = DPFC_CTL_PLANE(crtc->plane); if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) - dev_priv->fbc.threshold++; + threshold++; - switch (dev_priv->fbc.threshold) { + switch (threshold) { case 4: case 3: dpfc_ctl |= DPFC_CTL_LIMIT_4X; @@ -208,65 +258,61 @@ static void ilk_fbc_enable(struct drm_crtc *crtc) break; } dpfc_ctl |= DPFC_CTL_FENCE_EN; - if (IS_GEN5(dev)) + if (IS_GEN5(dev_priv)) dpfc_ctl |= obj->fence_reg; - I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); + y_offset = get_crtc_fence_y_offset(crtc); + I915_WRITE(ILK_DPFC_FENCE_YOFF, y_offset); I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID); /* enable it... */ I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); - if (IS_GEN6(dev)) { + if (IS_GEN6(dev_priv)) { I915_WRITE(SNB_DPFC_CTL_SA, SNB_CPU_FENCE_ENABLE | obj->fence_reg); - I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y); + I915_WRITE(DPFC_CPU_FENCE_OFFSET, y_offset); } - intel_fbc_nuke(dev_priv); - - DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); + intel_fbc_recompress(dev_priv); } -static void ilk_fbc_disable(struct drm_device *dev) +static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 dpfc_ctl; - dev_priv->fbc.enabled = false; + dev_priv->fbc.active = false; /* Disable compression */ dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); if (dpfc_ctl & DPFC_CTL_EN) { dpfc_ctl &= ~DPFC_CTL_EN; I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl); - - DRM_DEBUG_KMS("disabled FBC\n"); } } -static bool ilk_fbc_enabled(struct drm_device *dev) +static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN; } -static void gen7_fbc_enable(struct drm_crtc *crtc) +static void gen7_fbc_activate(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->primary->fb; + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_framebuffer *fb = crtc->base.primary->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 dpfc_ctl; + int threshold = dev_priv->fbc.threshold; - dev_priv->fbc.enabled = true; + dev_priv->fbc.active = true; + + dpfc_ctl = 0; + if (IS_IVYBRIDGE(dev_priv)) + dpfc_ctl |= IVB_DPFC_CTL_PLANE(crtc->plane); - dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane); if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) - dev_priv->fbc.threshold++; + threshold++; - switch (dev_priv->fbc.threshold) { + switch (threshold) { case 4: case 3: dpfc_ctl |= DPFC_CTL_LIMIT_4X; @@ -284,295 +330,450 @@ static void gen7_fbc_enable(struct drm_crtc *crtc) if (dev_priv->fbc.false_color) dpfc_ctl |= FBC_CTL_FALSE_COLOR; - I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); - - if (IS_IVYBRIDGE(dev)) { + if (IS_IVYBRIDGE(dev_priv)) { /* WaFbcAsynchFlipDisableFbcQueue:ivb */ I915_WRITE(ILK_DISPLAY_CHICKEN1, I915_READ(ILK_DISPLAY_CHICKEN1) | ILK_FBCQ_DIS); - } else { + } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ - I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe), - I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) | + I915_WRITE(CHICKEN_PIPESL_1(crtc->pipe), + I915_READ(CHICKEN_PIPESL_1(crtc->pipe)) | HSW_FBCQ_DIS); } + I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); + I915_WRITE(SNB_DPFC_CTL_SA, SNB_CPU_FENCE_ENABLE | obj->fence_reg); - I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y); + I915_WRITE(DPFC_CPU_FENCE_OFFSET, get_crtc_fence_y_offset(crtc)); - intel_fbc_nuke(dev_priv); - - DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); + intel_fbc_recompress(dev_priv); } /** - * intel_fbc_enabled - Is FBC enabled? - * @dev: the drm_device + * intel_fbc_is_active - Is FBC active? + * @dev_priv: i915 device instance * * This function is used to verify the current state of FBC. * FIXME: This should be tracked in the plane config eventually * instead of queried at runtime for most callers. */ -bool intel_fbc_enabled(struct drm_device *dev) +bool intel_fbc_is_active(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + return dev_priv->fbc.active; +} + +static void intel_fbc_activate(const struct drm_framebuffer *fb) +{ + struct drm_i915_private *dev_priv = fb->dev->dev_private; + struct intel_crtc *crtc = dev_priv->fbc.crtc; + + dev_priv->fbc.activate(crtc); - return dev_priv->fbc.enabled; + dev_priv->fbc.fb_id = fb->base.id; + dev_priv->fbc.y = crtc->base.y; } static void intel_fbc_work_fn(struct work_struct *__work) { - struct intel_fbc_work *work = - container_of(to_delayed_work(__work), - struct intel_fbc_work, work); - struct drm_device *dev = work->crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = + container_of(__work, struct drm_i915_private, fbc.work.work); + struct intel_fbc_work *work = &dev_priv->fbc.work; + struct intel_crtc *crtc = dev_priv->fbc.crtc; + int delay_ms = 50; - mutex_lock(&dev->struct_mutex); - if (work == dev_priv->fbc.fbc_work) { - /* Double check that we haven't switched fb without cancelling - * the prior work. - */ - if (work->crtc->primary->fb == work->fb) { - dev_priv->display.enable_fbc(work->crtc); +retry: + /* Delay the actual enabling to let pageflipping cease and the + * display to settle before starting the compression. Note that + * this delay also serves a second purpose: it allows for a + * vblank to pass after disabling the FBC before we attempt + * to modify the control registers. + * + * A more complicated solution would involve tracking vblanks + * following the termination of the page-flipping sequence + * and indeed performing the enable as a co-routine and not + * waiting synchronously upon the vblank. + * + * WaFbcWaitForVBlankBeforeEnable:ilk,snb + */ + wait_remaining_ms_from_jiffies(work->enable_jiffies, delay_ms); - dev_priv->fbc.crtc = to_intel_crtc(work->crtc); - dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id; - dev_priv->fbc.y = work->crtc->y; - } + mutex_lock(&dev_priv->fbc.lock); + + /* Were we cancelled? */ + if (!work->scheduled) + goto out; - dev_priv->fbc.fbc_work = NULL; + /* Were we delayed again while this function was sleeping? */ + if (time_after(work->enable_jiffies + msecs_to_jiffies(delay_ms), + jiffies)) { + mutex_unlock(&dev_priv->fbc.lock); + goto retry; } - mutex_unlock(&dev->struct_mutex); - kfree(work); + if (crtc->base.primary->fb == work->fb) + intel_fbc_activate(work->fb); + + work->scheduled = false; + +out: + mutex_unlock(&dev_priv->fbc.lock); } static void intel_fbc_cancel_work(struct drm_i915_private *dev_priv) { - if (dev_priv->fbc.fbc_work == NULL) - return; + WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock)); + dev_priv->fbc.work.scheduled = false; +} - DRM_DEBUG_KMS("cancelling pending FBC enable\n"); +static void intel_fbc_schedule_activation(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct intel_fbc_work *work = &dev_priv->fbc.work; - /* Synchronisation is provided by struct_mutex and checking of - * dev_priv->fbc.fbc_work, so we can perform the cancellation - * entirely asynchronously. - */ - if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work)) - /* tasklet was killed before being run, clean up */ - kfree(dev_priv->fbc.fbc_work); - - /* Mark the work as no longer wanted so that if it does - * wake-up (because the work was already running and waiting - * for our mutex), it will discover that is no longer - * necessary to run. - */ - dev_priv->fbc.fbc_work = NULL; + WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock)); + + /* It is useless to call intel_fbc_cancel_work() in this function since + * we're not releasing fbc.lock, so it won't have an opportunity to grab + * it to discover that it was cancelled. So we just update the expected + * jiffy count. */ + work->fb = crtc->base.primary->fb; + work->scheduled = true; + work->enable_jiffies = jiffies; + + schedule_work(&work->work); } -static void intel_fbc_enable(struct drm_crtc *crtc) +static void __intel_fbc_deactivate(struct drm_i915_private *dev_priv) { - struct intel_fbc_work *work; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock)); + + intel_fbc_cancel_work(dev_priv); + + if (dev_priv->fbc.active) + dev_priv->fbc.deactivate(dev_priv); +} + +/* + * intel_fbc_deactivate - deactivate FBC if it's associated with crtc + * @crtc: the CRTC + * + * This function deactivates FBC if it's associated with the provided CRTC. + */ +void intel_fbc_deactivate(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; - if (!dev_priv->display.enable_fbc) + if (!fbc_supported(dev_priv)) return; - intel_fbc_cancel_work(dev_priv); + mutex_lock(&dev_priv->fbc.lock); + if (dev_priv->fbc.crtc == crtc) + __intel_fbc_deactivate(dev_priv); + mutex_unlock(&dev_priv->fbc.lock); +} - work = kzalloc(sizeof(*work), GFP_KERNEL); - if (work == NULL) { - DRM_ERROR("Failed to allocate FBC work structure\n"); - dev_priv->display.enable_fbc(crtc); +static void set_no_fbc_reason(struct drm_i915_private *dev_priv, + const char *reason) +{ + if (dev_priv->fbc.no_fbc_reason == reason) return; + + dev_priv->fbc.no_fbc_reason = reason; + DRM_DEBUG_KMS("Disabling FBC: %s\n", reason); +} + +static bool crtc_can_fbc(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + + if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A) + return false; + + if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A) + return false; + + return true; +} + +static bool crtc_is_valid(struct intel_crtc *crtc) +{ + if (!intel_crtc_active(&crtc->base)) + return false; + + if (!to_intel_plane_state(crtc->base.primary->state)->visible) + return false; + + return true; +} + +static bool multiple_pipes_ok(struct drm_i915_private *dev_priv) +{ + enum pipe pipe; + int n_pipes = 0; + struct drm_crtc *crtc; + + if (INTEL_INFO(dev_priv)->gen > 4) + return true; + + for_each_pipe(dev_priv, pipe) { + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + + if (intel_crtc_active(crtc) && + to_intel_plane_state(crtc->primary->state)->visible) + n_pipes++; } - work->crtc = crtc; - work->fb = crtc->primary->fb; - INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn); + return (n_pipes < 2); +} - dev_priv->fbc.fbc_work = work; +static int find_compression_threshold(struct drm_i915_private *dev_priv, + struct drm_mm_node *node, + int size, + int fb_cpp) +{ + int compression_threshold = 1; + int ret; + u64 end; + + /* The FBC hardware for BDW/SKL doesn't have access to the stolen + * reserved range size, so it always assumes the maximum (8mb) is used. + * If we enable FBC using a CFB on that memory range we'll get FIFO + * underruns, even if that range is not reserved by the BIOS. */ + if (IS_BROADWELL(dev_priv) || + IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + end = dev_priv->gtt.stolen_size - 8 * 1024 * 1024; + else + end = dev_priv->gtt.stolen_usable_size; - /* Delay the actual enabling to let pageflipping cease and the - * display to settle before starting the compression. Note that - * this delay also serves a second purpose: it allows for a - * vblank to pass after disabling the FBC before we attempt - * to modify the control registers. + /* HACK: This code depends on what we will do in *_enable_fbc. If that + * code changes, this code needs to change as well. * - * A more complicated solution would involve tracking vblanks - * following the termination of the page-flipping sequence - * and indeed performing the enable as a co-routine and not - * waiting synchronously upon the vblank. - * - * WaFbcWaitForVBlankBeforeEnable:ilk,snb + * The enable_fbc code will attempt to use one of our 2 compression + * thresholds, therefore, in that case, we only have 1 resort. */ - schedule_delayed_work(&work->work, msecs_to_jiffies(50)); + + /* Try to over-allocate to reduce reallocations and fragmentation. */ + ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size <<= 1, + 4096, 0, end); + if (ret == 0) + return compression_threshold; + +again: + /* HW's ability to limit the CFB is 1:4 */ + if (compression_threshold > 4 || + (fb_cpp == 2 && compression_threshold == 2)) + return 0; + + ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1, + 4096, 0, end); + if (ret && INTEL_INFO(dev_priv)->gen <= 4) { + return 0; + } else if (ret) { + compression_threshold <<= 1; + goto again; + } else { + return compression_threshold; + } } -/** - * intel_fbc_disable - disable FBC - * @dev: the drm_device - * - * This function disables FBC. - */ -void intel_fbc_disable(struct drm_device *dev) +static int intel_fbc_alloc_cfb(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_framebuffer *fb = crtc->base.primary->state->fb; + struct drm_mm_node *uninitialized_var(compressed_llb); + int size, fb_cpp, ret; - intel_fbc_cancel_work(dev_priv); + WARN_ON(drm_mm_node_allocated(&dev_priv->fbc.compressed_fb)); + + size = intel_fbc_calculate_cfb_size(crtc, fb); + fb_cpp = drm_format_plane_cpp(fb->pixel_format, 0); + + ret = find_compression_threshold(dev_priv, &dev_priv->fbc.compressed_fb, + size, fb_cpp); + if (!ret) + goto err_llb; + else if (ret > 1) { + DRM_INFO("Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n"); + + } + + dev_priv->fbc.threshold = ret; + + if (INTEL_INFO(dev_priv)->gen >= 5) + I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->fbc.compressed_fb.start); + else if (IS_GM45(dev_priv)) { + I915_WRITE(DPFC_CB_BASE, dev_priv->fbc.compressed_fb.start); + } else { + compressed_llb = kzalloc(sizeof(*compressed_llb), GFP_KERNEL); + if (!compressed_llb) + goto err_fb; + + ret = i915_gem_stolen_insert_node(dev_priv, compressed_llb, + 4096, 4096); + if (ret) + goto err_fb; + + dev_priv->fbc.compressed_llb = compressed_llb; + + I915_WRITE(FBC_CFB_BASE, + dev_priv->mm.stolen_base + dev_priv->fbc.compressed_fb.start); + I915_WRITE(FBC_LL_BASE, + dev_priv->mm.stolen_base + compressed_llb->start); + } + + DRM_DEBUG_KMS("reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n", + dev_priv->fbc.compressed_fb.size, + dev_priv->fbc.threshold); + + return 0; + +err_fb: + kfree(compressed_llb); + i915_gem_stolen_remove_node(dev_priv, &dev_priv->fbc.compressed_fb); +err_llb: + pr_info_once("drm: not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size); + return -ENOSPC; +} + +static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv) +{ + if (drm_mm_node_allocated(&dev_priv->fbc.compressed_fb)) + i915_gem_stolen_remove_node(dev_priv, + &dev_priv->fbc.compressed_fb); + + if (dev_priv->fbc.compressed_llb) { + i915_gem_stolen_remove_node(dev_priv, + dev_priv->fbc.compressed_llb); + kfree(dev_priv->fbc.compressed_llb); + } +} - if (!dev_priv->display.disable_fbc) +void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv) +{ + if (!fbc_supported(dev_priv)) return; - dev_priv->display.disable_fbc(dev); - dev_priv->fbc.crtc = NULL; + mutex_lock(&dev_priv->fbc.lock); + __intel_fbc_cleanup_cfb(dev_priv); + mutex_unlock(&dev_priv->fbc.lock); } -static bool set_no_fbc_reason(struct drm_i915_private *dev_priv, - enum no_fbc_reason reason) +static bool stride_is_valid(struct drm_i915_private *dev_priv, + unsigned int stride) { - if (dev_priv->fbc.no_fbc_reason == reason) + /* These should have been caught earlier. */ + WARN_ON(stride < 512); + WARN_ON((stride & (64 - 1)) != 0); + + /* Below are the additional FBC restrictions. */ + + if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv)) + return stride == 4096 || stride == 8192; + + if (IS_GEN4(dev_priv) && !IS_G4X(dev_priv) && stride < 2048) + return false; + + if (stride > 16384) return false; - dev_priv->fbc.no_fbc_reason = reason; return true; } -static struct drm_crtc *intel_fbc_find_crtc(struct drm_i915_private *dev_priv) +static bool pixel_format_is_valid(struct drm_framebuffer *fb) { - struct drm_crtc *crtc = NULL, *tmp_crtc; - enum pipe pipe; - bool pipe_a_only = false, one_pipe_only = false; - - if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8) - pipe_a_only = true; - else if (INTEL_INFO(dev_priv)->gen <= 4) - one_pipe_only = true; - - for_each_pipe(dev_priv, pipe) { - tmp_crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - - if (intel_crtc_active(tmp_crtc) && - to_intel_plane_state(tmp_crtc->primary->state)->visible) { - if (one_pipe_only && crtc) { - if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES)) - DRM_DEBUG_KMS("more than one pipe active, disabling compression\n"); - return NULL; - } - crtc = tmp_crtc; - } + struct drm_device *dev = fb->dev; + struct drm_i915_private *dev_priv = dev->dev_private; - if (pipe_a_only) - break; + switch (fb->pixel_format) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_XBGR8888: + return true; + case DRM_FORMAT_XRGB1555: + case DRM_FORMAT_RGB565: + /* 16bpp not supported on gen2 */ + if (IS_GEN2(dev)) + return false; + /* WaFbcOnly1to1Ratio:ctg */ + if (IS_G4X(dev_priv)) + return false; + return true; + default: + return false; } +} - if (!crtc || crtc->primary->fb == NULL) { - if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT)) - DRM_DEBUG_KMS("no output, disabling\n"); - return NULL; +/* + * For some reason, the hardware tracking starts looking at whatever we + * programmed as the display plane base address register. It does not look at + * the X and Y offset registers. That's why we look at the crtc->adjusted{x,y} + * variables instead of just looking at the pipe/plane size. + */ +static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + unsigned int effective_w, effective_h, max_w, max_h; + + if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) { + max_w = 4096; + max_h = 4096; + } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) { + max_w = 4096; + max_h = 2048; + } else { + max_w = 2048; + max_h = 1536; } - return crtc; + intel_fbc_get_plane_source_size(crtc, &effective_w, &effective_h); + effective_w += crtc->adjusted_x; + effective_h += crtc->adjusted_y; + + return effective_w <= max_w && effective_h <= max_h; } /** - * intel_fbc_update - enable/disable FBC as needed - * @dev: the drm_device - * - * Set up the framebuffer compression hardware at mode set time. We - * enable it if possible: - * - plane A only (on pre-965) - * - no pixel mulitply/line duplication - * - no alpha buffer discard - * - no dual wide - * - framebuffer <= max_hdisplay in width, max_vdisplay in height + * __intel_fbc_update - activate/deactivate FBC as needed, unlocked + * @crtc: the CRTC that triggered the update * - * We can't assume that any compression will take place (worst case), - * so the compressed buffer has to be the same size as the uncompressed - * one. It also must reside (along with the line length buffer) in - * stolen memory. - * - * We need to enable/disable FBC on a global basis. + * This function completely reevaluates the status of FBC, then activates, + * deactivates or maintains it on the same state. */ -void intel_fbc_update(struct drm_device *dev) +static void __intel_fbc_update(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_crtc *crtc = NULL; - struct intel_crtc *intel_crtc; + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; struct drm_framebuffer *fb; struct drm_i915_gem_object *obj; const struct drm_display_mode *adjusted_mode; - unsigned int max_width, max_height; - if (!HAS_FBC(dev)) - return; + WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock)); - /* disable framebuffer compression in vGPU */ - if (intel_vgpu_active(dev)) - i915.enable_fbc = 0; - - if (i915.enable_fbc < 0) { - if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT)) - DRM_DEBUG_KMS("disabled per chip default\n"); + if (!multiple_pipes_ok(dev_priv)) { + set_no_fbc_reason(dev_priv, "more than one pipe active"); goto out_disable; } - if (!i915.enable_fbc) { - if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) - DRM_DEBUG_KMS("fbc disabled per module param\n"); - goto out_disable; - } + if (!dev_priv->fbc.enabled || dev_priv->fbc.crtc != crtc) + return; - /* - * If FBC is already on, we just have to verify that we can - * keep it that way... - * Need to disable if: - * - more than one pipe is active - * - changing FBC params (stride, fence, mode) - * - new fb is too large to fit in compressed buffer - * - going to an unsupported config (interlace, pixel multiply, etc.) - */ - crtc = intel_fbc_find_crtc(dev_priv); - if (!crtc) + if (!crtc_is_valid(crtc)) { + set_no_fbc_reason(dev_priv, "no output"); goto out_disable; + } - intel_crtc = to_intel_crtc(crtc); - fb = crtc->primary->fb; + fb = crtc->base.primary->fb; obj = intel_fb_obj(fb); - adjusted_mode = &intel_crtc->config->base.adjusted_mode; + adjusted_mode = &crtc->config->base.adjusted_mode; if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) || (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) { - if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE)) - DRM_DEBUG_KMS("mode incompatible with compression, " - "disabling\n"); + set_no_fbc_reason(dev_priv, "incompatible mode"); goto out_disable; } - if (INTEL_INFO(dev)->gen >= 8 || IS_HASWELL(dev)) { - max_width = 4096; - max_height = 4096; - } else if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { - max_width = 4096; - max_height = 2048; - } else { - max_width = 2048; - max_height = 1536; - } - if (intel_crtc->config->pipe_src_w > max_width || - intel_crtc->config->pipe_src_h > max_height) { - if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE)) - DRM_DEBUG_KMS("mode too large for compression, disabling\n"); - goto out_disable; - } - if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) && - intel_crtc->plane != PLANE_A) { - if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE)) - DRM_DEBUG_KMS("plane not A, disabling compression\n"); + if (!intel_fbc_hw_tracking_covers_screen(crtc)) { + set_no_fbc_reason(dev_priv, "mode too large for compression"); goto out_disable; } @@ -581,25 +782,46 @@ void intel_fbc_update(struct drm_device *dev) */ if (obj->tiling_mode != I915_TILING_X || obj->fence_reg == I915_FENCE_REG_NONE) { - if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED)) - DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n"); + set_no_fbc_reason(dev_priv, "framebuffer not tiled or fenced"); + goto out_disable; + } + if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) && + crtc->base.primary->state->rotation != BIT(DRM_ROTATE_0)) { + set_no_fbc_reason(dev_priv, "rotation unsupported"); + goto out_disable; + } + + if (!stride_is_valid(dev_priv, fb->pitches[0])) { + set_no_fbc_reason(dev_priv, "framebuffer stride not supported"); goto out_disable; } - if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) && - crtc->primary->state->rotation != BIT(DRM_ROTATE_0)) { - if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE)) - DRM_DEBUG_KMS("Rotation unsupported, disabling\n"); + + if (!pixel_format_is_valid(fb)) { + set_no_fbc_reason(dev_priv, "pixel format is invalid"); goto out_disable; } - /* If the kernel debugger is active, always disable compression */ - if (in_dbg_master()) + /* WaFbcExceedCdClockThreshold:hsw,bdw */ + if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) && + ilk_pipe_pixel_rate(crtc->config) >= + dev_priv->cdclk_freq * 95 / 100) { + set_no_fbc_reason(dev_priv, "pixel rate is too big"); goto out_disable; + } - if (i915_gem_stolen_setup_compression(dev, obj->base.size, - drm_format_plane_cpp(fb->pixel_format, 0))) { - if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL)) - DRM_DEBUG_KMS("framebuffer too large, disabling compression\n"); + /* It is possible for the required CFB size change without a + * crtc->disable + crtc->enable since it is possible to change the + * stride without triggering a full modeset. Since we try to + * over-allocate the CFB, there's a chance we may keep FBC enabled even + * if this happens, but if we exceed the current CFB size we'll have to + * disable FBC. Notice that it would be possible to disable FBC, wait + * for a frame, free the stolen node, then try to reenable FBC in case + * we didn't get any invalidate/deactivate calls, but this would require + * a lot of tracking just for a specific case. If we conclude it's an + * important case, we can implement it later. */ + if (intel_fbc_calculate_cfb_size(crtc, fb) > + dev_priv->fbc.compressed_fb.size * dev_priv->fbc.threshold) { + set_no_fbc_reason(dev_priv, "CFB requirements changed"); goto out_disable; } @@ -608,12 +830,13 @@ void intel_fbc_update(struct drm_device *dev) * cannot be unpinned (and have its GTT offset and fence revoked) * without first being decoupled from the scanout and FBC disabled. */ - if (dev_priv->fbc.crtc == intel_crtc && + if (dev_priv->fbc.crtc == crtc && dev_priv->fbc.fb_id == fb->base.id && - dev_priv->fbc.y == crtc->y) + dev_priv->fbc.y == crtc->base.y && + dev_priv->fbc.active) return; - if (intel_fbc_enabled(dev)) { + if (intel_fbc_is_active(dev_priv)) { /* We update FBC along two paths, after changing fb/crtc * configuration (modeswitching) and after page-flipping * finishes. For the latter, we know that not only did @@ -637,59 +860,211 @@ void intel_fbc_update(struct drm_device *dev) * disabling paths we do need to wait for a vblank at * some point. And we wait before enabling FBC anyway. */ - DRM_DEBUG_KMS("disabling active FBC for update\n"); - intel_fbc_disable(dev); + DRM_DEBUG_KMS("deactivating FBC for update\n"); + __intel_fbc_deactivate(dev_priv); } - intel_fbc_enable(crtc); - dev_priv->fbc.no_fbc_reason = FBC_OK; + intel_fbc_schedule_activation(crtc); + dev_priv->fbc.no_fbc_reason = "FBC enabled (not necessarily active)"; return; out_disable: /* Multiple disables should be harmless */ - if (intel_fbc_enabled(dev)) { - DRM_DEBUG_KMS("unsupported config, disabling FBC\n"); - intel_fbc_disable(dev); + if (intel_fbc_is_active(dev_priv)) { + DRM_DEBUG_KMS("unsupported config, deactivating FBC\n"); + __intel_fbc_deactivate(dev_priv); } - i915_gem_stolen_cleanup_compression(dev); +} + +/* + * intel_fbc_update - activate/deactivate FBC as needed + * @crtc: the CRTC that triggered the update + * + * This function reevaluates the overall state and activates or deactivates FBC. + */ +void intel_fbc_update(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + + if (!fbc_supported(dev_priv)) + return; + + mutex_lock(&dev_priv->fbc.lock); + __intel_fbc_update(crtc); + mutex_unlock(&dev_priv->fbc.lock); } void intel_fbc_invalidate(struct drm_i915_private *dev_priv, unsigned int frontbuffer_bits, enum fb_op_origin origin) { - struct drm_device *dev = dev_priv->dev; unsigned int fbc_bits; + if (!fbc_supported(dev_priv)) + return; + if (origin == ORIGIN_GTT) return; + mutex_lock(&dev_priv->fbc.lock); + if (dev_priv->fbc.enabled) fbc_bits = INTEL_FRONTBUFFER_PRIMARY(dev_priv->fbc.crtc->pipe); - else if (dev_priv->fbc.fbc_work) - fbc_bits = INTEL_FRONTBUFFER_PRIMARY( - to_intel_crtc(dev_priv->fbc.fbc_work->crtc)->pipe); else fbc_bits = dev_priv->fbc.possible_framebuffer_bits; dev_priv->fbc.busy_bits |= (fbc_bits & frontbuffer_bits); if (dev_priv->fbc.busy_bits) - intel_fbc_disable(dev); + __intel_fbc_deactivate(dev_priv); + + mutex_unlock(&dev_priv->fbc.lock); } void intel_fbc_flush(struct drm_i915_private *dev_priv, - unsigned int frontbuffer_bits) + unsigned int frontbuffer_bits, enum fb_op_origin origin) { - struct drm_device *dev = dev_priv->dev; + if (!fbc_supported(dev_priv)) + return; - if (!dev_priv->fbc.busy_bits) + if (origin == ORIGIN_GTT) return; + mutex_lock(&dev_priv->fbc.lock); + dev_priv->fbc.busy_bits &= ~frontbuffer_bits; - if (!dev_priv->fbc.busy_bits) - intel_fbc_update(dev); + if (!dev_priv->fbc.busy_bits && dev_priv->fbc.enabled) { + if (origin != ORIGIN_FLIP && dev_priv->fbc.active) { + intel_fbc_recompress(dev_priv); + } else { + __intel_fbc_deactivate(dev_priv); + __intel_fbc_update(dev_priv->fbc.crtc); + } + } + + mutex_unlock(&dev_priv->fbc.lock); +} + +/** + * intel_fbc_enable: tries to enable FBC on the CRTC + * @crtc: the CRTC + * + * This function checks if it's possible to enable FBC on the following CRTC, + * then enables it. Notice that it doesn't activate FBC. + */ +void intel_fbc_enable(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + + if (!fbc_supported(dev_priv)) + return; + + mutex_lock(&dev_priv->fbc.lock); + + if (dev_priv->fbc.enabled) { + WARN_ON(dev_priv->fbc.crtc == crtc); + goto out; + } + + WARN_ON(dev_priv->fbc.active); + WARN_ON(dev_priv->fbc.crtc != NULL); + + if (intel_vgpu_active(dev_priv->dev)) { + set_no_fbc_reason(dev_priv, "VGPU is active"); + goto out; + } + + if (i915.enable_fbc < 0) { + set_no_fbc_reason(dev_priv, "disabled per chip default"); + goto out; + } + + if (!i915.enable_fbc) { + set_no_fbc_reason(dev_priv, "disabled per module param"); + goto out; + } + + if (!crtc_can_fbc(crtc)) { + set_no_fbc_reason(dev_priv, "no enabled pipes can have FBC"); + goto out; + } + + if (intel_fbc_alloc_cfb(crtc)) { + set_no_fbc_reason(dev_priv, "not enough stolen memory"); + goto out; + } + + DRM_DEBUG_KMS("Enabling FBC on pipe %c\n", pipe_name(crtc->pipe)); + dev_priv->fbc.no_fbc_reason = "FBC enabled but not active yet\n"; + + dev_priv->fbc.enabled = true; + dev_priv->fbc.crtc = crtc; +out: + mutex_unlock(&dev_priv->fbc.lock); +} + +/** + * __intel_fbc_disable - disable FBC + * @dev_priv: i915 device instance + * + * This is the low level function that actually disables FBC. Callers should + * grab the FBC lock. + */ +static void __intel_fbc_disable(struct drm_i915_private *dev_priv) +{ + struct intel_crtc *crtc = dev_priv->fbc.crtc; + + WARN_ON(!mutex_is_locked(&dev_priv->fbc.lock)); + WARN_ON(!dev_priv->fbc.enabled); + WARN_ON(dev_priv->fbc.active); + assert_pipe_disabled(dev_priv, crtc->pipe); + + DRM_DEBUG_KMS("Disabling FBC on pipe %c\n", pipe_name(crtc->pipe)); + + __intel_fbc_cleanup_cfb(dev_priv); + + dev_priv->fbc.enabled = false; + dev_priv->fbc.crtc = NULL; +} + +/** + * intel_fbc_disable_crtc - disable FBC if it's associated with crtc + * @crtc: the CRTC + * + * This function disables FBC if it's associated with the provided CRTC. + */ +void intel_fbc_disable_crtc(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + + if (!fbc_supported(dev_priv)) + return; + + mutex_lock(&dev_priv->fbc.lock); + if (dev_priv->fbc.crtc == crtc) { + WARN_ON(!dev_priv->fbc.enabled); + WARN_ON(dev_priv->fbc.active); + __intel_fbc_disable(dev_priv); + } + mutex_unlock(&dev_priv->fbc.lock); +} + +/** + * intel_fbc_disable - globally disable FBC + * @dev_priv: i915 device instance + * + * This function disables FBC regardless of which CRTC is associated with it. + */ +void intel_fbc_disable(struct drm_i915_private *dev_priv) +{ + if (!fbc_supported(dev_priv)) + return; + + mutex_lock(&dev_priv->fbc.lock); + if (dev_priv->fbc.enabled) + __intel_fbc_disable(dev_priv); + mutex_unlock(&dev_priv->fbc.lock); } /** @@ -702,9 +1077,14 @@ void intel_fbc_init(struct drm_i915_private *dev_priv) { enum pipe pipe; + INIT_WORK(&dev_priv->fbc.work.work, intel_fbc_work_fn); + mutex_init(&dev_priv->fbc.lock); + dev_priv->fbc.enabled = false; + dev_priv->fbc.active = false; + dev_priv->fbc.work.scheduled = false; + if (!HAS_FBC(dev_priv)) { - dev_priv->fbc.enabled = false; - dev_priv->fbc.no_fbc_reason = FBC_UNSUPPORTED; + dev_priv->fbc.no_fbc_reason = "unsupported by this chipset"; return; } @@ -712,30 +1092,34 @@ void intel_fbc_init(struct drm_i915_private *dev_priv) dev_priv->fbc.possible_framebuffer_bits |= INTEL_FRONTBUFFER_PRIMARY(pipe); - if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8) + if (fbc_on_pipe_a_only(dev_priv)) break; } if (INTEL_INFO(dev_priv)->gen >= 7) { - dev_priv->display.fbc_enabled = ilk_fbc_enabled; - dev_priv->display.enable_fbc = gen7_fbc_enable; - dev_priv->display.disable_fbc = ilk_fbc_disable; + dev_priv->fbc.is_active = ilk_fbc_is_active; + dev_priv->fbc.activate = gen7_fbc_activate; + dev_priv->fbc.deactivate = ilk_fbc_deactivate; } else if (INTEL_INFO(dev_priv)->gen >= 5) { - dev_priv->display.fbc_enabled = ilk_fbc_enabled; - dev_priv->display.enable_fbc = ilk_fbc_enable; - dev_priv->display.disable_fbc = ilk_fbc_disable; + dev_priv->fbc.is_active = ilk_fbc_is_active; + dev_priv->fbc.activate = ilk_fbc_activate; + dev_priv->fbc.deactivate = ilk_fbc_deactivate; } else if (IS_GM45(dev_priv)) { - dev_priv->display.fbc_enabled = g4x_fbc_enabled; - dev_priv->display.enable_fbc = g4x_fbc_enable; - dev_priv->display.disable_fbc = g4x_fbc_disable; + dev_priv->fbc.is_active = g4x_fbc_is_active; + dev_priv->fbc.activate = g4x_fbc_activate; + dev_priv->fbc.deactivate = g4x_fbc_deactivate; } else { - dev_priv->display.fbc_enabled = i8xx_fbc_enabled; - dev_priv->display.enable_fbc = i8xx_fbc_enable; - dev_priv->display.disable_fbc = i8xx_fbc_disable; + dev_priv->fbc.is_active = i8xx_fbc_is_active; + dev_priv->fbc.activate = i8xx_fbc_activate; + dev_priv->fbc.deactivate = i8xx_fbc_deactivate; /* This value was pulled out of someone's hat */ I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); } - dev_priv->fbc.enabled = dev_priv->display.fbc_enabled(dev_priv->dev); + /* We still don't have any sort of hardware state readout for FBC, so + * deactivate it in case the BIOS activated it to make sure software + * matches the hardware state. */ + if (dev_priv->fbc.is_active(dev_priv)) + dev_priv->fbc.deactivate(dev_priv); } diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index 6372cfc7d053..7ccde58f8c98 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -55,16 +55,8 @@ static int intel_fbdev_set_par(struct fb_info *info) ret = drm_fb_helper_set_par(info); if (ret == 0) { - /* - * FIXME: fbdev presumes that all callbacks also work from - * atomic contexts and relies on that for emergency oops - * printing. KMS totally doesn't do that and the locking here is - * by far not the only place this goes wrong. Ignore this for - * now until we solve this for real. - */ mutex_lock(&fb_helper->dev->struct_mutex); - ret = i915_gem_object_set_to_gtt_domain(ifbdev->fb->obj, - true); + intel_fb_obj_invalidate(ifbdev->fb->obj, ORIGIN_GTT); mutex_unlock(&fb_helper->dev->struct_mutex); } @@ -81,15 +73,8 @@ static int intel_fbdev_blank(int blank, struct fb_info *info) ret = drm_fb_helper_blank(blank, info); if (ret == 0) { - /* - * FIXME: fbdev presumes that all callbacks also work from - * atomic contexts and relies on that for emergency oops - * printing. KMS totally doesn't do that and the locking here is - * by far not the only place this goes wrong. Ignore this for - * now until we solve this for real. - */ mutex_lock(&fb_helper->dev->struct_mutex); - intel_fb_obj_invalidate(ifbdev->fb->obj, NULL, ORIGIN_GTT); + intel_fb_obj_invalidate(ifbdev->fb->obj, ORIGIN_GTT); mutex_unlock(&fb_helper->dev->struct_mutex); } @@ -107,15 +92,8 @@ static int intel_fbdev_pan_display(struct fb_var_screeninfo *var, ret = drm_fb_helper_pan_display(var, info); if (ret == 0) { - /* - * FIXME: fbdev presumes that all callbacks also work from - * atomic contexts and relies on that for emergency oops - * printing. KMS totally doesn't do that and the locking here is - * by far not the only place this goes wrong. Ignore this for - * now until we solve this for real. - */ mutex_lock(&fb_helper->dev->struct_mutex); - intel_fb_obj_invalidate(ifbdev->fb->obj, NULL, ORIGIN_GTT); + intel_fb_obj_invalidate(ifbdev->fb->obj, ORIGIN_GTT); mutex_unlock(&fb_helper->dev->struct_mutex); } @@ -126,9 +104,9 @@ static struct fb_ops intelfb_ops = { .owner = THIS_MODULE, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = intel_fbdev_set_par, - .fb_fillrect = cfb_fillrect, - .fb_copyarea = cfb_copyarea, - .fb_imageblit = cfb_imageblit, + .fb_fillrect = drm_fb_helper_cfb_fillrect, + .fb_copyarea = drm_fb_helper_cfb_copyarea, + .fb_imageblit = drm_fb_helper_cfb_imageblit, .fb_pan_display = intel_fbdev_pan_display, .fb_blank = intel_fbdev_blank, .fb_setcmap = drm_fb_helper_setcmap, @@ -141,10 +119,11 @@ static int intelfb_alloc(struct drm_fb_helper *helper, { struct intel_fbdev *ifbdev = container_of(helper, struct intel_fbdev, helper); - struct drm_framebuffer *fb; + struct drm_framebuffer *fb = NULL; struct drm_device *dev = helper->dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_mode_fb_cmd2 mode_cmd = {}; - struct drm_i915_gem_object *obj; + struct drm_i915_gem_object *obj = NULL; int size, ret; /* we don't do packed 24bpp */ @@ -159,9 +138,16 @@ static int intelfb_alloc(struct drm_fb_helper *helper, mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth); + mutex_lock(&dev->struct_mutex); + size = mode_cmd.pitches[0] * mode_cmd.height; size = PAGE_ALIGN(size); - obj = i915_gem_object_create_stolen(dev, size); + + /* If the FB is too big, just don't use it since fbdev is not very + * important and we should probably use that space with FBC or other + * features. */ + if (size * 2 < dev_priv->gtt.stolen_usable_size) + obj = i915_gem_object_create_stolen(dev, size); if (obj == NULL) obj = i915_gem_alloc_object(dev, size); if (!obj) { @@ -172,26 +158,28 @@ static int intelfb_alloc(struct drm_fb_helper *helper, fb = __intel_framebuffer_create(dev, &mode_cmd, obj); if (IS_ERR(fb)) { + drm_gem_object_unreference(&obj->base); ret = PTR_ERR(fb); - goto out_unref; + goto out; } /* Flush everything out, we'll be doing GTT only from now on */ - ret = intel_pin_and_fence_fb_obj(NULL, fb, NULL, NULL); + ret = intel_pin_and_fence_fb_obj(NULL, fb, NULL); if (ret) { DRM_ERROR("failed to pin obj: %d\n", ret); - goto out_fb; + goto out; } + mutex_unlock(&dev->struct_mutex); + ifbdev->fb = to_intel_framebuffer(fb); return 0; -out_fb: - drm_framebuffer_remove(fb); -out_unref: - drm_gem_object_unreference(&obj->base); out: + mutex_unlock(&dev->struct_mutex); + if (!IS_ERR_OR_NULL(fb)) + drm_framebuffer_unreference(fb); return ret; } @@ -209,8 +197,6 @@ static int intelfb_create(struct drm_fb_helper *helper, int size, ret; bool prealloc = false; - mutex_lock(&dev->struct_mutex); - if (intel_fb && (sizes->fb_width > intel_fb->base.width || sizes->fb_height > intel_fb->base.height)) { @@ -225,7 +211,7 @@ static int intelfb_create(struct drm_fb_helper *helper, DRM_DEBUG_KMS("no BIOS fb, allocating a new one\n"); ret = intelfb_alloc(helper, sizes); if (ret) - goto out_unlock; + return ret; intel_fb = ifbdev->fb; } else { DRM_DEBUG_KMS("re-using BIOS fb\n"); @@ -237,9 +223,12 @@ static int intelfb_create(struct drm_fb_helper *helper, obj = intel_fb->obj; size = obj->base.size; - info = framebuffer_alloc(0, &dev->pdev->dev); - if (!info) { - ret = -ENOMEM; + mutex_lock(&dev->struct_mutex); + + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) { + DRM_ERROR("Failed to allocate fb_info\n"); + ret = PTR_ERR(info); goto out_unpin; } @@ -248,24 +237,13 @@ static int intelfb_create(struct drm_fb_helper *helper, fb = &ifbdev->fb->base; ifbdev->helper.fb = fb; - ifbdev->helper.fbdev = info; strcpy(info->fix.id, "inteldrmfb"); info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; info->fbops = &intelfb_ops; - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - ret = -ENOMEM; - goto out_unpin; - } /* setup aperture base/size for vesafb takeover */ - info->apertures = alloc_apertures(1); - if (!info->apertures) { - ret = -ENOMEM; - goto out_unpin; - } info->apertures->ranges[0].base = dev->mode_config.fb_base; info->apertures->ranges[0].size = dev_priv->gtt.mappable_end; @@ -276,8 +254,9 @@ static int intelfb_create(struct drm_fb_helper *helper, ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj), size); if (!info->screen_base) { + DRM_ERROR("Failed to remap framebuffer into virtual memory\n"); ret = -ENOSPC; - goto out_unpin; + goto out_destroy_fbi; } info->screen_size = size; @@ -296,7 +275,7 @@ static int intelfb_create(struct drm_fb_helper *helper, /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ - DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08lx, bo %p\n", + DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08llx, bo %p\n", fb->width, fb->height, i915_gem_obj_ggtt_offset(obj), obj); @@ -304,10 +283,10 @@ static int intelfb_create(struct drm_fb_helper *helper, vga_switcheroo_client_fb_set(dev->pdev, info); return 0; +out_destroy_fbi: + drm_fb_helper_release_fbi(helper); out_unpin: i915_gem_object_ggtt_unpin(obj); - drm_gem_object_unreference(&obj->base); -out_unlock: mutex_unlock(&dev->struct_mutex); return ret; } @@ -484,18 +463,13 @@ retry: * IMPORTANT: We want to use the adjusted mode (i.e. * after the panel fitter upscaling) as the initial * config, not the input mode, which is what crtc->mode - * usually contains. But since our current fastboot + * usually contains. But since our current * code puts a mode derived from the post-pfit timings - * into crtc->mode this works out correctly. We don't - * use hwmode anywhere right now, so use it for this - * since the fb helper layer wants a pointer to - * something we own. + * into crtc->mode this works out correctly. */ DRM_DEBUG_KMS("looking for current mode on connector %s\n", connector->name); - intel_mode_from_pipe_config(&encoder->crtc->hwmode, - to_intel_crtc(encoder->crtc)->config); - modes[i] = &encoder->crtc->hwmode; + modes[i] = &encoder->crtc->mode; } crtcs[i] = new_crtc; @@ -550,21 +524,16 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = { static void intel_fbdev_destroy(struct drm_device *dev, struct intel_fbdev *ifbdev) { - if (ifbdev->helper.fbdev) { - struct fb_info *info = ifbdev->helper.fbdev; - - unregister_framebuffer(info); - iounmap(info->screen_base); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(&ifbdev->helper); + drm_fb_helper_release_fbi(&ifbdev->helper); drm_fb_helper_fini(&ifbdev->helper); - drm_framebuffer_unregister_private(&ifbdev->fb->base); - drm_framebuffer_remove(&ifbdev->fb->base); + if (ifbdev->fb) { + drm_framebuffer_unregister_private(&ifbdev->fb->base); + drm_framebuffer_remove(&ifbdev->fb->base); + } } /* @@ -582,28 +551,25 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, struct intel_framebuffer *fb = NULL; struct drm_crtc *crtc; struct intel_crtc *intel_crtc; - struct intel_initial_plane_config *plane_config = NULL; unsigned int max_size = 0; - if (!i915.fastboot) - return false; - /* Find the largest fb */ for_each_crtc(dev, crtc) { + struct drm_i915_gem_object *obj = + intel_fb_obj(crtc->primary->state->fb); intel_crtc = to_intel_crtc(crtc); - if (!intel_crtc->active || !crtc->primary->fb) { + if (!crtc->state->active || !obj) { DRM_DEBUG_KMS("pipe %c not active or no fb, skipping\n", pipe_name(intel_crtc->pipe)); continue; } - if (intel_crtc->plane_config.size > max_size) { + if (obj->base.size > max_size) { DRM_DEBUG_KMS("found possible fb from plane %c\n", pipe_name(intel_crtc->pipe)); - plane_config = &intel_crtc->plane_config; - fb = to_intel_framebuffer(crtc->primary->fb); - max_size = plane_config->size; + fb = to_intel_framebuffer(crtc->primary->state->fb); + max_size = obj->base.size; } } @@ -618,7 +584,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, intel_crtc = to_intel_crtc(crtc); - if (!intel_crtc->active) { + if (!crtc->state->active) { DRM_DEBUG_KMS("pipe %c not active, skipping\n", pipe_name(intel_crtc->pipe)); continue; @@ -638,7 +604,6 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, DRM_DEBUG_KMS("fb not wide enough for plane %c (%d vs %d)\n", pipe_name(intel_crtc->pipe), cur_size, fb->base.pitches[0]); - plane_config = NULL; fb = NULL; break; } @@ -659,7 +624,6 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, DRM_DEBUG_KMS("fb not big enough for plane %c (%d vs %d)\n", pipe_name(intel_crtc->pipe), cur_size, max_size); - plane_config = NULL; fb = NULL; break; } @@ -683,7 +647,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, for_each_crtc(dev, crtc) { intel_crtc = to_intel_crtc(crtc); - if (!intel_crtc->active) + if (!crtc->state->active) continue; WARN(!crtc->primary->fb, @@ -734,6 +698,8 @@ int intel_fbdev_init(struct drm_device *dev) return ret; } + ifbdev->helper.atomic = true; + dev_priv->fbdev = ifbdev; INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker); @@ -742,13 +708,20 @@ int intel_fbdev_init(struct drm_device *dev) return 0; } -void intel_fbdev_initial_config(void *data, async_cookie_t cookie) +static void intel_fbdev_initial_config(void *data, async_cookie_t cookie) { struct drm_i915_private *dev_priv = data; struct intel_fbdev *ifbdev = dev_priv->fbdev; /* Due to peculiar init order wrt to hpd handling this is separate. */ - drm_fb_helper_initial_config(&ifbdev->helper, ifbdev->preferred_bpp); + if (drm_fb_helper_initial_config(&ifbdev->helper, + ifbdev->preferred_bpp)) + intel_fbdev_fini(dev_priv->dev); +} + +void intel_fbdev_initial_config_async(struct drm_device *dev) +{ + async_schedule(intel_fbdev_initial_config, to_i915(dev)); } void intel_fbdev_fini(struct drm_device *dev) @@ -759,7 +732,8 @@ void intel_fbdev_fini(struct drm_device *dev) flush_work(&dev_priv->fbdev_suspend_work); - async_synchronize_full(); + if (!current_is_async()) + async_synchronize_full(); intel_fbdev_destroy(dev, dev_priv->fbdev); kfree(dev_priv->fbdev); dev_priv->fbdev = NULL; @@ -810,7 +784,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous if (state == FBINFO_STATE_RUNNING && ifbdev->fb->obj->stolen) memset_io(info->screen_base, 0, info->screen_size); - fb_set_suspend(info, state); + drm_fb_helper_set_suspend(&ifbdev->helper, state); console_unlock(); } @@ -825,11 +799,20 @@ void intel_fbdev_restore_mode(struct drm_device *dev) { int ret; struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_fbdev *ifbdev = dev_priv->fbdev; + struct drm_fb_helper *fb_helper; - if (!dev_priv->fbdev) + if (!ifbdev) return; - ret = drm_fb_helper_restore_fbdev_mode_unlocked(&dev_priv->fbdev->helper); - if (ret) + fb_helper = &ifbdev->helper; + + ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper); + if (ret) { DRM_DEBUG("failed to restore crtc mode\n"); + } else { + mutex_lock(&fb_helper->dev->struct_mutex); + intel_fb_obj_invalidate(ifbdev->fb->obj, ORIGIN_GTT); + mutex_unlock(&fb_helper->dev->struct_mutex); + } } diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 54daa66c6970..bda526660e20 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -84,38 +84,21 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) return true; } -/** - * i9xx_check_fifo_underruns - check for fifo underruns - * @dev_priv: i915 device instance - * - * This function checks for fifo underruns on GMCH platforms. This needs to be - * done manually on modeset to make sure that we catch all underruns since they - * do not generate an interrupt by themselves on these platforms. - */ -void i9xx_check_fifo_underruns(struct drm_i915_private *dev_priv) +static void i9xx_check_fifo_underruns(struct intel_crtc *crtc) { - struct intel_crtc *crtc; - - spin_lock_irq(&dev_priv->irq_lock); - - for_each_intel_crtc(dev_priv->dev, crtc) { - u32 reg = PIPESTAT(crtc->pipe); - u32 pipestat; - - if (crtc->cpu_fifo_underrun_disabled) - continue; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + i915_reg_t reg = PIPESTAT(crtc->pipe); + u32 pipestat = I915_READ(reg) & 0xffff0000; - pipestat = I915_READ(reg) & 0xffff0000; - if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0) - continue; + assert_spin_locked(&dev_priv->irq_lock); - I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS); - POSTING_READ(reg); + if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0) + return; - DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe)); - } + I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS); + POSTING_READ(reg); - spin_unlock_irq(&dev_priv->irq_lock); + DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe)); } static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, @@ -123,7 +106,7 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, bool enable, bool old) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg = PIPESTAT(pipe); + i915_reg_t reg = PIPESTAT(pipe); u32 pipestat = I915_READ(reg) & 0xffff0000; assert_spin_locked(&dev_priv->irq_lock); @@ -145,9 +128,26 @@ static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev, DE_PIPEB_FIFO_UNDERRUN; if (enable) - ironlake_enable_display_irq(dev_priv, bit); + ilk_enable_display_irq(dev_priv, bit); else - ironlake_disable_display_irq(dev_priv, bit); + ilk_disable_display_irq(dev_priv, bit); +} + +static void ivybridge_check_fifo_underruns(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + uint32_t err_int = I915_READ(GEN7_ERR_INT); + + assert_spin_locked(&dev_priv->irq_lock); + + if ((err_int & ERR_INT_FIFO_UNDERRUN(pipe)) == 0) + return; + + I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe)); + POSTING_READ(GEN7_ERR_INT); + + DRM_ERROR("fifo underrun on pipe %c\n", pipe_name(pipe)); } static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev, @@ -161,9 +161,9 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev, if (!ivb_can_enable_err_int(dev)) return; - ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB); + ilk_enable_display_irq(dev_priv, DE_ERR_INT_IVB); } else { - ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB); + ilk_disable_display_irq(dev_priv, DE_ERR_INT_IVB); if (old && I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe)) { @@ -178,14 +178,10 @@ static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev, { struct drm_i915_private *dev_priv = dev->dev_private; - assert_spin_locked(&dev_priv->irq_lock); - if (enable) - dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_FIFO_UNDERRUN; + bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN); else - dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_FIFO_UNDERRUN; - I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); - POSTING_READ(GEN8_DE_PIPE_IMR(pipe)); + bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN); } static void ibx_set_fifo_underrun_reporting(struct drm_device *dev, @@ -202,6 +198,24 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev, ibx_disable_display_interrupt(dev_priv, bit); } +static void cpt_check_pch_fifo_underruns(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum transcoder pch_transcoder = (enum transcoder) crtc->pipe; + uint32_t serr_int = I915_READ(SERR_INT); + + assert_spin_locked(&dev_priv->irq_lock); + + if ((serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) == 0) + return; + + I915_WRITE(SERR_INT, SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)); + POSTING_READ(SERR_INT); + + DRM_ERROR("pch fifo underrun on pch transcoder %c\n", + transcoder_name(pch_transcoder)); +} + static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, enum transcoder pch_transcoder, bool enable, bool old) @@ -375,3 +389,56 @@ void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, DRM_ERROR("PCH transcoder %c FIFO underrun\n", transcoder_name(pch_transcoder)); } + +/** + * intel_check_cpu_fifo_underruns - check for CPU fifo underruns immediately + * @dev_priv: i915 device instance + * + * Check for CPU fifo underruns immediately. Useful on IVB/HSW where the shared + * error interrupt may have been disabled, and so CPU fifo underruns won't + * necessarily raise an interrupt, and on GMCH platforms where underruns never + * raise an interrupt. + */ +void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv) +{ + struct intel_crtc *crtc; + + spin_lock_irq(&dev_priv->irq_lock); + + for_each_intel_crtc(dev_priv->dev, crtc) { + if (crtc->cpu_fifo_underrun_disabled) + continue; + + if (HAS_GMCH_DISPLAY(dev_priv)) + i9xx_check_fifo_underruns(crtc); + else if (IS_GEN7(dev_priv)) + ivybridge_check_fifo_underruns(crtc); + } + + spin_unlock_irq(&dev_priv->irq_lock); +} + +/** + * intel_check_pch_fifo_underruns - check for PCH fifo underruns immediately + * @dev_priv: i915 device instance + * + * Check for PCH fifo underruns immediately. Useful on CPT/PPT where the shared + * error interrupt may have been disabled, and so PCH fifo underruns won't + * necessarily raise an interrupt. + */ +void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv) +{ + struct intel_crtc *crtc; + + spin_lock_irq(&dev_priv->irq_lock); + + for_each_intel_crtc(dev_priv->dev, crtc) { + if (crtc->pch_fifo_underrun_disabled) + continue; + + if (HAS_PCH_CPT(dev_priv)) + cpt_check_pch_fifo_underruns(crtc); + } + + spin_unlock_irq(&dev_priv->irq_lock); +} diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.c b/drivers/gpu/drm/i915/intel_frontbuffer.c index 57095f54c1f2..ac85357010b4 100644 --- a/drivers/gpu/drm/i915/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/intel_frontbuffer.c @@ -65,84 +65,29 @@ #include "intel_drv.h" #include "i915_drv.h" -static void intel_increase_pllclock(struct drm_device *dev, - enum pipe pipe) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int dpll_reg = DPLL(pipe); - int dpll; - - if (!HAS_GMCH_DISPLAY(dev)) - return; - - if (!dev_priv->lvds_downclock_avail) - return; - - dpll = I915_READ(dpll_reg); - if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) { - DRM_DEBUG_DRIVER("upclocking LVDS\n"); - - assert_panel_unlocked(dev_priv, pipe); - - dpll &= ~DISPLAY_RATE_SELECT_FPA1; - I915_WRITE(dpll_reg, dpll); - intel_wait_for_vblank(dev, pipe); - - dpll = I915_READ(dpll_reg); - if (dpll & DISPLAY_RATE_SELECT_FPA1) - DRM_DEBUG_DRIVER("failed to upclock LVDS!\n"); - } -} - -/** - * intel_mark_fb_busy - mark given planes as busy - * @dev: DRM device - * @frontbuffer_bits: bits for the affected planes - * @ring: optional ring for asynchronous commands - * - * This function gets called every time the screen contents change. It can be - * used to keep e.g. the update rate at the nominal refresh rate with DRRS. - */ -static void intel_mark_fb_busy(struct drm_device *dev, - unsigned frontbuffer_bits, - struct intel_engine_cs *ring) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - enum pipe pipe; - - for_each_pipe(dev_priv, pipe) { - if (!(frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(pipe))) - continue; - - intel_increase_pllclock(dev, pipe); - } -} - /** * intel_fb_obj_invalidate - invalidate frontbuffer object * @obj: GEM object to invalidate - * @ring: set for asynchronous rendering * @origin: which operation caused the invalidation * * This function gets called every time rendering on the given object starts and * frontbuffer caching (fbc, low refresh rate for DRRS, panel self refresh) must - * be invalidated. If @ring is non-NULL any subsequent invalidation will be delayed + * be invalidated. For ORIGIN_CS any subsequent invalidation will be delayed * until the rendering completes or a flip on this frontbuffer plane is * scheduled. */ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, - struct intel_engine_cs *ring, enum fb_op_origin origin) { struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); WARN_ON(!mutex_is_locked(&dev->struct_mutex)); if (!obj->frontbuffer_bits) return; - if (ring) { + if (origin == ORIGIN_CS) { mutex_lock(&dev_priv->fb_tracking.lock); dev_priv->fb_tracking.busy_bits |= obj->frontbuffer_bits; @@ -151,8 +96,6 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, mutex_unlock(&dev_priv->fb_tracking.lock); } - intel_mark_fb_busy(dev, obj->frontbuffer_bits, ring); - intel_psr_invalidate(dev, obj->frontbuffer_bits); intel_edp_drrs_invalidate(dev, obj->frontbuffer_bits); intel_fbc_invalidate(dev_priv, obj->frontbuffer_bits, origin); @@ -162,6 +105,7 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, * intel_frontbuffer_flush - flush frontbuffer * @dev: DRM device * @frontbuffer_bits: frontbuffer plane tracking bits + * @origin: which operation caused the flush * * This function gets called every time rendering on the given planes has * completed and frontbuffer caching can be started again. Flushes will get @@ -169,37 +113,40 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, * * Can be called without any locks held. */ -void intel_frontbuffer_flush(struct drm_device *dev, - unsigned frontbuffer_bits) +static void intel_frontbuffer_flush(struct drm_device *dev, + unsigned frontbuffer_bits, + enum fb_op_origin origin) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* Delay flushing when rings are still busy.*/ mutex_lock(&dev_priv->fb_tracking.lock); frontbuffer_bits &= ~dev_priv->fb_tracking.busy_bits; mutex_unlock(&dev_priv->fb_tracking.lock); - intel_mark_fb_busy(dev, frontbuffer_bits, NULL); + if (!frontbuffer_bits) + return; intel_edp_drrs_flush(dev, frontbuffer_bits); - intel_psr_flush(dev, frontbuffer_bits); - intel_fbc_flush(dev_priv, frontbuffer_bits); + intel_psr_flush(dev, frontbuffer_bits, origin); + intel_fbc_flush(dev_priv, frontbuffer_bits, origin); } /** * intel_fb_obj_flush - flush frontbuffer object * @obj: GEM object to flush * @retire: set when retiring asynchronous rendering + * @origin: which operation caused the flush * * This function gets called every time rendering on the given object has * completed and frontbuffer caching can be started again. If @retire is true * then any delayed flushes will be unblocked. */ void intel_fb_obj_flush(struct drm_i915_gem_object *obj, - bool retire) + bool retire, enum fb_op_origin origin) { struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned frontbuffer_bits; WARN_ON(!mutex_is_locked(&dev->struct_mutex)); @@ -218,7 +165,7 @@ void intel_fb_obj_flush(struct drm_i915_gem_object *obj, mutex_unlock(&dev_priv->fb_tracking.lock); } - intel_frontbuffer_flush(dev, frontbuffer_bits); + intel_frontbuffer_flush(dev, frontbuffer_bits, origin); } /** @@ -236,7 +183,7 @@ void intel_fb_obj_flush(struct drm_i915_gem_object *obj, void intel_frontbuffer_flip_prepare(struct drm_device *dev, unsigned frontbuffer_bits) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); mutex_lock(&dev_priv->fb_tracking.lock); dev_priv->fb_tracking.flip_bits |= frontbuffer_bits; @@ -244,7 +191,7 @@ void intel_frontbuffer_flip_prepare(struct drm_device *dev, dev_priv->fb_tracking.busy_bits &= ~frontbuffer_bits; mutex_unlock(&dev_priv->fb_tracking.lock); - intel_psr_single_frame_update(dev); + intel_psr_single_frame_update(dev, frontbuffer_bits); } /** @@ -260,7 +207,7 @@ void intel_frontbuffer_flip_prepare(struct drm_device *dev, void intel_frontbuffer_flip_complete(struct drm_device *dev, unsigned frontbuffer_bits) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); mutex_lock(&dev_priv->fb_tracking.lock); /* Mask any cancelled flips. */ @@ -268,5 +215,29 @@ void intel_frontbuffer_flip_complete(struct drm_device *dev, dev_priv->fb_tracking.flip_bits &= ~frontbuffer_bits; mutex_unlock(&dev_priv->fb_tracking.lock); - intel_frontbuffer_flush(dev, frontbuffer_bits); + intel_frontbuffer_flush(dev, frontbuffer_bits, ORIGIN_FLIP); +} + +/** + * intel_frontbuffer_flip - synchronous frontbuffer flip + * @dev: DRM device + * @frontbuffer_bits: frontbuffer plane tracking bits + * + * This function gets called after scheduling a flip on @obj. This is for + * synchronous plane updates which will happen on the next vblank and which will + * not get delayed by pending gpu rendering. + * + * Can be called without any locks held. + */ +void intel_frontbuffer_flip(struct drm_device *dev, + unsigned frontbuffer_bits) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + + mutex_lock(&dev_priv->fb_tracking.lock); + /* Remove stale busy bits due to the old buffer. */ + dev_priv->fb_tracking.busy_bits &= ~frontbuffer_bits; + mutex_unlock(&dev_priv->fb_tracking.lock); + + intel_frontbuffer_flush(dev, frontbuffer_bits, ORIGIN_FLIP); } diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h new file mode 100644 index 000000000000..822952235dcf --- /dev/null +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -0,0 +1,126 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ +#ifndef _INTEL_GUC_H_ +#define _INTEL_GUC_H_ + +#include "intel_guc_fwif.h" +#include "i915_guc_reg.h" + +struct i915_guc_client { + struct drm_i915_gem_object *client_obj; + struct intel_context *owner; + struct intel_guc *guc; + uint32_t priority; + uint32_t ctx_index; + + uint32_t proc_desc_offset; + uint32_t doorbell_offset; + uint32_t cookie; + uint16_t doorbell_id; + uint16_t padding; /* Maintain alignment */ + + uint32_t wq_offset; + uint32_t wq_size; + uint32_t wq_tail; + + /* GuC submission statistics & status */ + uint64_t submissions[I915_NUM_RINGS]; + uint32_t q_fail; + uint32_t b_fail; + int retcode; +}; + +enum intel_guc_fw_status { + GUC_FIRMWARE_FAIL = -1, + GUC_FIRMWARE_NONE = 0, + GUC_FIRMWARE_PENDING, + GUC_FIRMWARE_SUCCESS +}; + +/* + * This structure encapsulates all the data needed during the process + * of fetching, caching, and loading the firmware image into the GuC. + */ +struct intel_guc_fw { + struct drm_device * guc_dev; + const char * guc_fw_path; + size_t guc_fw_size; + struct drm_i915_gem_object * guc_fw_obj; + enum intel_guc_fw_status guc_fw_fetch_status; + enum intel_guc_fw_status guc_fw_load_status; + + uint16_t guc_fw_major_wanted; + uint16_t guc_fw_minor_wanted; + uint16_t guc_fw_major_found; + uint16_t guc_fw_minor_found; + + uint32_t header_size; + uint32_t header_offset; + uint32_t rsa_size; + uint32_t rsa_offset; + uint32_t ucode_size; + uint32_t ucode_offset; +}; + +struct intel_guc { + struct intel_guc_fw guc_fw; + uint32_t log_flags; + struct drm_i915_gem_object *log_obj; + + struct drm_i915_gem_object *ctx_pool_obj; + struct ida ctx_ids; + + struct i915_guc_client *execbuf_client; + + DECLARE_BITMAP(doorbell_bitmap, GUC_MAX_DOORBELLS); + uint32_t db_cacheline; /* Cyclic counter mod pagesize */ + + /* Action status & statistics */ + uint64_t action_count; /* Total commands issued */ + uint32_t action_cmd; /* Last command word */ + uint32_t action_status; /* Last return status */ + uint32_t action_fail; /* Total number of failures */ + int32_t action_err; /* Last error code */ + + uint64_t submissions[I915_NUM_RINGS]; + uint32_t last_seqno[I915_NUM_RINGS]; +}; + +/* intel_guc_loader.c */ +extern void intel_guc_ucode_init(struct drm_device *dev); +extern int intel_guc_ucode_load(struct drm_device *dev); +extern void intel_guc_ucode_fini(struct drm_device *dev); +extern const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status); +extern int intel_guc_suspend(struct drm_device *dev); +extern int intel_guc_resume(struct drm_device *dev); + +/* i915_guc_submission.c */ +int i915_guc_submission_init(struct drm_device *dev); +int i915_guc_submission_enable(struct drm_device *dev); +int i915_guc_submit(struct i915_guc_client *client, + struct drm_i915_gem_request *rq); +void i915_guc_submission_disable(struct drm_device *dev); +void i915_guc_submission_fini(struct drm_device *dev); + +#endif diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h new file mode 100644 index 000000000000..40b2ea572e16 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_guc_fwif.h @@ -0,0 +1,332 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#ifndef _INTEL_GUC_FWIF_H +#define _INTEL_GUC_FWIF_H + +/* + * This file is partially autogenerated, although currently with some manual + * fixups afterwards. In future, it should be entirely autogenerated, in order + * to ensure that the definitions herein remain in sync with those used by the + * GuC's own firmware. + * + * EDITING THIS FILE IS THEREFORE NOT RECOMMENDED - YOUR CHANGES MAY BE LOST. + */ + +#define GFXCORE_FAMILY_GEN9 12 +#define GFXCORE_FAMILY_UNKNOWN 0x7fffffff + +#define GUC_CTX_PRIORITY_KMD_HIGH 0 +#define GUC_CTX_PRIORITY_HIGH 1 +#define GUC_CTX_PRIORITY_KMD_NORMAL 2 +#define GUC_CTX_PRIORITY_NORMAL 3 + +#define GUC_MAX_GPU_CONTEXTS 1024 +#define GUC_INVALID_CTX_ID GUC_MAX_GPU_CONTEXTS + +/* Work queue item header definitions */ +#define WQ_STATUS_ACTIVE 1 +#define WQ_STATUS_SUSPENDED 2 +#define WQ_STATUS_CMD_ERROR 3 +#define WQ_STATUS_ENGINE_ID_NOT_USED 4 +#define WQ_STATUS_SUSPENDED_FROM_RESET 5 +#define WQ_TYPE_SHIFT 0 +#define WQ_TYPE_BATCH_BUF (0x1 << WQ_TYPE_SHIFT) +#define WQ_TYPE_PSEUDO (0x2 << WQ_TYPE_SHIFT) +#define WQ_TYPE_INORDER (0x3 << WQ_TYPE_SHIFT) +#define WQ_TARGET_SHIFT 10 +#define WQ_LEN_SHIFT 16 +#define WQ_NO_WCFLUSH_WAIT (1 << 27) +#define WQ_PRESENT_WORKLOAD (1 << 28) +#define WQ_WORKLOAD_SHIFT 29 +#define WQ_WORKLOAD_GENERAL (0 << WQ_WORKLOAD_SHIFT) +#define WQ_WORKLOAD_GPGPU (1 << WQ_WORKLOAD_SHIFT) +#define WQ_WORKLOAD_TOUCH (2 << WQ_WORKLOAD_SHIFT) + +#define WQ_RING_TAIL_SHIFT 20 +#define WQ_RING_TAIL_MASK (0x7FF << WQ_RING_TAIL_SHIFT) + +#define GUC_DOORBELL_ENABLED 1 +#define GUC_DOORBELL_DISABLED 0 + +#define GUC_CTX_DESC_ATTR_ACTIVE (1 << 0) +#define GUC_CTX_DESC_ATTR_PENDING_DB (1 << 1) +#define GUC_CTX_DESC_ATTR_KERNEL (1 << 2) +#define GUC_CTX_DESC_ATTR_PREEMPT (1 << 3) +#define GUC_CTX_DESC_ATTR_RESET (1 << 4) +#define GUC_CTX_DESC_ATTR_WQLOCKED (1 << 5) +#define GUC_CTX_DESC_ATTR_PCH (1 << 6) +#define GUC_CTX_DESC_ATTR_TERMINATED (1 << 7) + +/* The guc control data is 10 DWORDs */ +#define GUC_CTL_CTXINFO 0 +#define GUC_CTL_CTXNUM_IN16_SHIFT 0 +#define GUC_CTL_BASE_ADDR_SHIFT 12 +#define GUC_CTL_ARAT_HIGH 1 +#define GUC_CTL_ARAT_LOW 2 +#define GUC_CTL_DEVICE_INFO 3 +#define GUC_CTL_GTTYPE_SHIFT 0 +#define GUC_CTL_COREFAMILY_SHIFT 7 +#define GUC_CTL_LOG_PARAMS 4 +#define GUC_LOG_VALID (1 << 0) +#define GUC_LOG_NOTIFY_ON_HALF_FULL (1 << 1) +#define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) +#define GUC_LOG_CRASH_PAGES 1 +#define GUC_LOG_CRASH_SHIFT 4 +#define GUC_LOG_DPC_PAGES 3 +#define GUC_LOG_DPC_SHIFT 6 +#define GUC_LOG_ISR_PAGES 3 +#define GUC_LOG_ISR_SHIFT 9 +#define GUC_LOG_BUF_ADDR_SHIFT 12 +#define GUC_CTL_PAGE_FAULT_CONTROL 5 +#define GUC_CTL_WA 6 +#define GUC_CTL_WA_UK_BY_DRIVER (1 << 3) +#define GUC_CTL_FEATURE 7 +#define GUC_CTL_VCS2_ENABLED (1 << 0) +#define GUC_CTL_KERNEL_SUBMISSIONS (1 << 1) +#define GUC_CTL_FEATURE2 (1 << 2) +#define GUC_CTL_POWER_GATING (1 << 3) +#define GUC_CTL_DISABLE_SCHEDULER (1 << 4) +#define GUC_CTL_PREEMPTION_LOG (1 << 5) +#define GUC_CTL_ENABLE_SLPC (1 << 7) +#define GUC_CTL_RESET_ON_PREMPT_FAILURE (1 << 8) +#define GUC_CTL_DEBUG 8 +#define GUC_LOG_VERBOSITY_SHIFT 0 +#define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_MED (1 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_HIGH (2 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_ULTRA (3 << GUC_LOG_VERBOSITY_SHIFT) +/* Verbosity range-check limits, without the shift */ +#define GUC_LOG_VERBOSITY_MIN 0 +#define GUC_LOG_VERBOSITY_MAX 3 +#define GUC_CTL_RSRVD 9 + +#define GUC_CTL_MAX_DWORDS (GUC_CTL_RSRVD + 1) + +/** + * DOC: GuC Firmware Layout + * + * The GuC firmware layout looks like this: + * + * +-------------------------------+ + * | guc_css_header | + * | contains major/minor version | + * +-------------------------------+ + * | uCode | + * +-------------------------------+ + * | RSA signature | + * +-------------------------------+ + * | modulus key | + * +-------------------------------+ + * | exponent val | + * +-------------------------------+ + * + * The firmware may or may not have modulus key and exponent data. The header, + * uCode and RSA signature are must-have components that will be used by driver. + * Length of each components, which is all in dwords, can be found in header. + * In the case that modulus and exponent are not present in fw, a.k.a truncated + * image, the length value still appears in header. + * + * Driver will do some basic fw size validation based on the following rules: + * + * 1. Header, uCode and RSA are must-have components. + * 2. All firmware components, if they present, are in the sequence illustrated + * in the layout table above. + * 3. Length info of each component can be found in header, in dwords. + * 4. Modulus and exponent key are not required by driver. They may not appear + * in fw. So driver will load a truncated firmware in this case. + */ + +struct guc_css_header { + uint32_t module_type; + /* header_size includes all non-uCode bits, including css_header, rsa + * key, modulus key and exponent data. */ + uint32_t header_size_dw; + uint32_t header_version; + uint32_t module_id; + uint32_t module_vendor; + union { + struct { + uint8_t day; + uint8_t month; + uint16_t year; + }; + uint32_t date; + }; + uint32_t size_dw; /* uCode plus header_size_dw */ + uint32_t key_size_dw; + uint32_t modulus_size_dw; + uint32_t exponent_size_dw; + union { + struct { + uint8_t hour; + uint8_t min; + uint16_t sec; + }; + uint32_t time; + }; + + char username[8]; + char buildnumber[12]; + uint32_t device_id; + uint32_t guc_sw_version; + uint32_t prod_preprod_fw; + uint32_t reserved[12]; + uint32_t header_info; +} __packed; + +struct guc_doorbell_info { + u32 db_status; + u32 cookie; + u32 reserved[14]; +} __packed; + +union guc_doorbell_qw { + struct { + u32 db_status; + u32 cookie; + }; + u64 value_qw; +} __packed; + +#define GUC_MAX_DOORBELLS 256 +#define GUC_INVALID_DOORBELL_ID (GUC_MAX_DOORBELLS) + +#define GUC_DB_SIZE (PAGE_SIZE) +#define GUC_WQ_SIZE (PAGE_SIZE * 2) + +/* Work item for submitting workloads into work queue of GuC. */ +struct guc_wq_item { + u32 header; + u32 context_desc; + u32 ring_tail; + u32 fence_id; +} __packed; + +struct guc_process_desc { + u32 context_id; + u64 db_base_addr; + u32 head; + u32 tail; + u32 error_offset; + u64 wq_base_addr; + u32 wq_size_bytes; + u32 wq_status; + u32 engine_presence; + u32 priority; + u32 reserved[30]; +} __packed; + +/* engine id and context id is packed into guc_execlist_context.context_id*/ +#define GUC_ELC_CTXID_OFFSET 0 +#define GUC_ELC_ENGINE_OFFSET 29 + +/* The execlist context including software and HW information */ +struct guc_execlist_context { + u32 context_desc; + u32 context_id; + u32 ring_status; + u32 ring_lcra; + u32 ring_begin; + u32 ring_end; + u32 ring_next_free_location; + u32 ring_current_tail_pointer_value; + u8 engine_state_submit_value; + u8 engine_state_wait_value; + u16 pagefault_count; + u16 engine_submit_queue_count; +} __packed; + +/*Context descriptor for communicating between uKernel and Driver*/ +struct guc_context_desc { + u32 sched_common_area; + u32 context_id; + u32 pas_id; + u8 engines_used; + u64 db_trigger_cpu; + u32 db_trigger_uk; + u64 db_trigger_phy; + u16 db_id; + + struct guc_execlist_context lrc[I915_NUM_RINGS]; + + u8 attribute; + + u32 priority; + + u32 wq_sampled_tail_offset; + u32 wq_total_submit_enqueues; + + u32 process_desc; + u32 wq_addr; + u32 wq_size; + + u32 engine_presence; + + u8 engine_suspended; + + u8 reserved0[3]; + u64 reserved1[1]; + + u64 desc_private; +} __packed; + +#define GUC_FORCEWAKE_RENDER (1 << 0) +#define GUC_FORCEWAKE_MEDIA (1 << 1) + +#define GUC_POWER_UNSPECIFIED 0 +#define GUC_POWER_D0 1 +#define GUC_POWER_D1 2 +#define GUC_POWER_D2 3 +#define GUC_POWER_D3 4 + +/* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */ +enum host2guc_action { + HOST2GUC_ACTION_DEFAULT = 0x0, + HOST2GUC_ACTION_SAMPLE_FORCEWAKE = 0x6, + HOST2GUC_ACTION_ALLOCATE_DOORBELL = 0x10, + HOST2GUC_ACTION_DEALLOCATE_DOORBELL = 0x20, + HOST2GUC_ACTION_ENTER_S_STATE = 0x501, + HOST2GUC_ACTION_EXIT_S_STATE = 0x502, + HOST2GUC_ACTION_SLPC_REQUEST = 0x3003, + HOST2GUC_ACTION_LIMIT +}; + +/* + * The GuC sends its response to a command by overwriting the + * command in SS0. The response is distinguishable from a command + * by the fact that all the MASK bits are set. The remaining bits + * give more detail. + */ +#define GUC2HOST_RESPONSE_MASK ((u32)0xF0000000) +#define GUC2HOST_IS_RESPONSE(x) ((u32)(x) >= GUC2HOST_RESPONSE_MASK) +#define GUC2HOST_STATUS(x) (GUC2HOST_RESPONSE_MASK | (x)) + +/* GUC will return status back to SOFT_SCRATCH_O_REG */ +enum guc2host_status { + GUC2HOST_STATUS_SUCCESS = GUC2HOST_STATUS(0x0), + GUC2HOST_STATUS_ALLOCATE_DOORBELL_FAIL = GUC2HOST_STATUS(0x10), + GUC2HOST_STATUS_DEALLOCATE_DOORBELL_FAIL = GUC2HOST_STATUS(0x20), + GUC2HOST_STATUS_GENERIC_FAIL = GUC2HOST_STATUS(0x0000F000) +}; + +#endif diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c new file mode 100644 index 000000000000..550921f2ef7d --- /dev/null +++ b/drivers/gpu/drm/i915/intel_guc_loader.c @@ -0,0 +1,637 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Vinit Azad <vinit.azad@intel.com> + * Ben Widawsky <ben@bwidawsk.net> + * Dave Gordon <david.s.gordon@intel.com> + * Alex Dai <yu.dai@intel.com> + */ +#include <linux/firmware.h> +#include "i915_drv.h" +#include "intel_guc.h" + +/** + * DOC: GuC-specific firmware loader + * + * intel_guc: + * Top level structure of guc. It handles firmware loading and manages client + * pool and doorbells. intel_guc owns a i915_guc_client to replace the legacy + * ExecList submission. + * + * Firmware versioning: + * The firmware build process will generate a version header file with major and + * minor version defined. The versions are built into CSS header of firmware. + * i915 kernel driver set the minimal firmware version required per platform. + * The firmware installation package will install (symbolic link) proper version + * of firmware. + * + * GuC address space: + * GuC does not allow any gfx GGTT address that falls into range [0, WOPCM_TOP), + * which is reserved for Boot ROM, SRAM and WOPCM. Currently this top address is + * 512K. In order to exclude 0-512K address space from GGTT, all gfx objects + * used by GuC is pinned with PIN_OFFSET_BIAS along with size of WOPCM. + * + * Firmware log: + * Firmware log is enabled by setting i915.guc_log_level to non-negative level. + * Log data is printed out via reading debugfs i915_guc_log_dump. Reading from + * i915_guc_load_status will print out firmware loading status and scratch + * registers value. + * + */ + +#define I915_SKL_GUC_UCODE "i915/skl_guc_ver4.bin" +MODULE_FIRMWARE(I915_SKL_GUC_UCODE); + +/* User-friendly representation of an enum */ +const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status) +{ + switch (status) { + case GUC_FIRMWARE_FAIL: + return "FAIL"; + case GUC_FIRMWARE_NONE: + return "NONE"; + case GUC_FIRMWARE_PENDING: + return "PENDING"; + case GUC_FIRMWARE_SUCCESS: + return "SUCCESS"; + default: + return "UNKNOWN!"; + } +}; + +static void direct_interrupts_to_host(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *ring; + int i, irqs; + + /* tell all command streamers NOT to forward interrupts and vblank to GuC */ + irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER); + irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING); + for_each_ring(ring, dev_priv, i) + I915_WRITE(RING_MODE_GEN7(ring), irqs); + + /* route all GT interrupts to the host */ + I915_WRITE(GUC_BCS_RCS_IER, 0); + I915_WRITE(GUC_VCS2_VCS1_IER, 0); + I915_WRITE(GUC_WD_VECS_IER, 0); +} + +static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *ring; + int i, irqs; + + /* tell all command streamers to forward interrupts and vblank to GuC */ + irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_ALWAYS); + irqs |= _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING); + for_each_ring(ring, dev_priv, i) + I915_WRITE(RING_MODE_GEN7(ring), irqs); + + /* route USER_INTERRUPT to Host, all others are sent to GuC. */ + irqs = GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT | + GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; + /* These three registers have the same bit definitions */ + I915_WRITE(GUC_BCS_RCS_IER, ~irqs); + I915_WRITE(GUC_VCS2_VCS1_IER, ~irqs); + I915_WRITE(GUC_WD_VECS_IER, ~irqs); +} + +static u32 get_gttype(struct drm_i915_private *dev_priv) +{ + /* XXX: GT type based on PCI device ID? field seems unused by fw */ + return 0; +} + +static u32 get_core_family(struct drm_i915_private *dev_priv) +{ + switch (INTEL_INFO(dev_priv)->gen) { + case 9: + return GFXCORE_FAMILY_GEN9; + + default: + DRM_ERROR("GUC: unsupported core family\n"); + return GFXCORE_FAMILY_UNKNOWN; + } +} + +static void set_guc_init_params(struct drm_i915_private *dev_priv) +{ + struct intel_guc *guc = &dev_priv->guc; + u32 params[GUC_CTL_MAX_DWORDS]; + int i; + + memset(¶ms, 0, sizeof(params)); + + params[GUC_CTL_DEVICE_INFO] |= + (get_gttype(dev_priv) << GUC_CTL_GTTYPE_SHIFT) | + (get_core_family(dev_priv) << GUC_CTL_COREFAMILY_SHIFT); + + /* + * GuC ARAT increment is 10 ns. GuC default scheduler quantum is one + * second. This ARAR is calculated by: + * Scheduler-Quantum-in-ns / ARAT-increment-in-ns = 1000000000 / 10 + */ + params[GUC_CTL_ARAT_HIGH] = 0; + params[GUC_CTL_ARAT_LOW] = 100000000; + + params[GUC_CTL_WA] |= GUC_CTL_WA_UK_BY_DRIVER; + + params[GUC_CTL_FEATURE] |= GUC_CTL_DISABLE_SCHEDULER | + GUC_CTL_VCS2_ENABLED; + + if (i915.guc_log_level >= 0) { + params[GUC_CTL_LOG_PARAMS] = guc->log_flags; + params[GUC_CTL_DEBUG] = + i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT; + } + + /* If GuC submission is enabled, set up additional parameters here */ + if (i915.enable_guc_submission) { + u32 pgs = i915_gem_obj_ggtt_offset(dev_priv->guc.ctx_pool_obj); + u32 ctx_in_16 = GUC_MAX_GPU_CONTEXTS / 16; + + pgs >>= PAGE_SHIFT; + params[GUC_CTL_CTXINFO] = (pgs << GUC_CTL_BASE_ADDR_SHIFT) | + (ctx_in_16 << GUC_CTL_CTXNUM_IN16_SHIFT); + + params[GUC_CTL_FEATURE] |= GUC_CTL_KERNEL_SUBMISSIONS; + + /* Unmask this bit to enable the GuC's internal scheduler */ + params[GUC_CTL_FEATURE] &= ~GUC_CTL_DISABLE_SCHEDULER; + } + + I915_WRITE(SOFT_SCRATCH(0), 0); + + for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) + I915_WRITE(SOFT_SCRATCH(1 + i), params[i]); +} + +/* + * Read the GuC status register (GUC_STATUS) and store it in the + * specified location; then return a boolean indicating whether + * the value matches either of two values representing completion + * of the GuC boot process. + * + * This is used for polling the GuC status in a wait_for_atomic() + * loop below. + */ +static inline bool guc_ucode_response(struct drm_i915_private *dev_priv, + u32 *status) +{ + u32 val = I915_READ(GUC_STATUS); + u32 uk_val = val & GS_UKERNEL_MASK; + *status = val; + return (uk_val == GS_UKERNEL_READY || + ((val & GS_MIA_CORE_STATE) && uk_val == GS_UKERNEL_LAPIC_DONE)); +} + +/* + * Transfer the firmware image to RAM for execution by the microcontroller. + * + * Architecturally, the DMA engine is bidirectional, and can potentially even + * transfer between GTT locations. This functionality is left out of the API + * for now as there is no need for it. + * + * Note that GuC needs the CSS header plus uKernel code to be copied by the + * DMA engine in one operation, whereas the RSA signature is loaded via MMIO. + */ +static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv) +{ + struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; + struct drm_i915_gem_object *fw_obj = guc_fw->guc_fw_obj; + unsigned long offset; + struct sg_table *sg = fw_obj->pages; + u32 status, rsa[UOS_RSA_SCRATCH_MAX_COUNT]; + int i, ret = 0; + + /* where RSA signature starts */ + offset = guc_fw->rsa_offset; + + /* Copy RSA signature from the fw image to HW for verification */ + sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa), offset); + for (i = 0; i < UOS_RSA_SCRATCH_MAX_COUNT; i++) + I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]); + + /* The header plus uCode will be copied to WOPCM via DMA, excluding any + * other components */ + I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size); + + /* Set the source address for the new blob */ + offset = i915_gem_obj_ggtt_offset(fw_obj) + guc_fw->header_offset; + I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset)); + I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF); + + /* + * Set the DMA destination. Current uCode expects the code to be + * loaded at 8k; locations below this are used for the stack. + */ + I915_WRITE(DMA_ADDR_1_LOW, 0x2000); + I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM); + + /* Finally start the DMA */ + I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(UOS_MOVE | START_DMA)); + + /* + * Spin-wait for the DMA to complete & the GuC to start up. + * NB: Docs recommend not using the interrupt for completion. + * Measurements indicate this should take no more than 20ms, so a + * timeout here indicates that the GuC has failed and is unusable. + * (Higher levels of the driver will attempt to fall back to + * execlist mode if this happens.) + */ + ret = wait_for_atomic(guc_ucode_response(dev_priv, &status), 100); + + DRM_DEBUG_DRIVER("DMA status 0x%x, GuC status 0x%x\n", + I915_READ(DMA_CTRL), status); + + if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) { + DRM_ERROR("GuC firmware signature verification failed\n"); + ret = -ENOEXEC; + } + + DRM_DEBUG_DRIVER("returning %d\n", ret); + + return ret; +} + +/* + * Load the GuC firmware blob into the MinuteIA. + */ +static int guc_ucode_xfer(struct drm_i915_private *dev_priv) +{ + struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; + struct drm_device *dev = dev_priv->dev; + int ret; + + ret = i915_gem_object_set_to_gtt_domain(guc_fw->guc_fw_obj, false); + if (ret) { + DRM_DEBUG_DRIVER("set-domain failed %d\n", ret); + return ret; + } + + ret = i915_gem_obj_ggtt_pin(guc_fw->guc_fw_obj, 0, 0); + if (ret) { + DRM_DEBUG_DRIVER("pin failed %d\n", ret); + return ret; + } + + /* Invalidate GuC TLB to let GuC take the latest updates to GTT. */ + I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); + + intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); + + /* init WOPCM */ + I915_WRITE(GUC_WOPCM_SIZE, GUC_WOPCM_SIZE_VALUE); + I915_WRITE(DMA_GUC_WOPCM_OFFSET, GUC_WOPCM_OFFSET_VALUE); + + /* Enable MIA caching. GuC clock gating is disabled. */ + I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE); + + /* WaDisableMinuteIaClockGating:skl,bxt */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) & + ~GUC_ENABLE_MIA_CLOCK_GATING)); + } + + /* WaC6DisallowByGfxPause*/ + I915_WRITE(GEN6_GFXPAUSE, 0x30FFF); + + if (IS_BROXTON(dev)) + I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE); + else + I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE); + + if (IS_GEN9(dev)) { + /* DOP Clock Gating Enable for GuC clocks */ + I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE | + I915_READ(GEN7_MISCCPCTL))); + + /* allows for 5us before GT can go to RC6 */ + I915_WRITE(GUC_ARAT_C6DIS, 0x1FF); + } + + set_guc_init_params(dev_priv); + + ret = guc_ucode_xfer_dma(dev_priv); + + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + + /* + * We keep the object pages for reuse during resume. But we can unpin it + * now that DMA has completed, so it doesn't continue to take up space. + */ + i915_gem_object_ggtt_unpin(guc_fw->guc_fw_obj); + + return ret; +} + +/** + * intel_guc_ucode_load() - load GuC uCode into the device + * @dev: drm device + * + * Called from gem_init_hw() during driver loading and also after a GPU reset. + * + * The firmware image should have already been fetched into memory by the + * earlier call to intel_guc_ucode_init(), so here we need only check that + * is succeeded, and then transfer the image to the h/w. + * + * Return: non-zero code on error + */ +int intel_guc_ucode_load(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; + int err = 0; + + if (!i915.enable_guc_submission) + return 0; + + DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status), + intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); + + direct_interrupts_to_host(dev_priv); + + if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_NONE) + return 0; + + if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_SUCCESS && + guc_fw->guc_fw_load_status == GUC_FIRMWARE_FAIL) + return -ENOEXEC; + + guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING; + + DRM_DEBUG_DRIVER("GuC fw fetch status %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status)); + + switch (guc_fw->guc_fw_fetch_status) { + case GUC_FIRMWARE_FAIL: + /* something went wrong :( */ + err = -EIO; + goto fail; + + case GUC_FIRMWARE_NONE: + case GUC_FIRMWARE_PENDING: + default: + /* "can't happen" */ + WARN_ONCE(1, "GuC fw %s invalid guc_fw_fetch_status %s [%d]\n", + guc_fw->guc_fw_path, + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status), + guc_fw->guc_fw_fetch_status); + err = -ENXIO; + goto fail; + + case GUC_FIRMWARE_SUCCESS: + break; + } + + err = i915_guc_submission_init(dev); + if (err) + goto fail; + + err = guc_ucode_xfer(dev_priv); + if (err) + goto fail; + + guc_fw->guc_fw_load_status = GUC_FIRMWARE_SUCCESS; + + DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status), + intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); + + if (i915.enable_guc_submission) { + /* The execbuf_client will be recreated. Release it first. */ + i915_guc_submission_disable(dev); + + err = i915_guc_submission_enable(dev); + if (err) + goto fail; + direct_interrupts_to_guc(dev_priv); + } + + return 0; + +fail: + if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_PENDING) + guc_fw->guc_fw_load_status = GUC_FIRMWARE_FAIL; + + direct_interrupts_to_host(dev_priv); + i915_guc_submission_disable(dev); + + return err; +} + +static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw) +{ + struct drm_i915_gem_object *obj; + const struct firmware *fw; + struct guc_css_header *css; + size_t size; + int err; + + DRM_DEBUG_DRIVER("before requesting firmware: GuC fw fetch status %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status)); + + err = request_firmware(&fw, guc_fw->guc_fw_path, &dev->pdev->dev); + if (err) + goto fail; + if (!fw) + goto fail; + + DRM_DEBUG_DRIVER("fetch GuC fw from %s succeeded, fw %p\n", + guc_fw->guc_fw_path, fw); + + /* Check the size of the blob before examining buffer contents */ + if (fw->size < sizeof(struct guc_css_header)) { + DRM_ERROR("Firmware header is missing\n"); + goto fail; + } + + css = (struct guc_css_header *)fw->data; + + /* Firmware bits always start from header */ + guc_fw->header_offset = 0; + guc_fw->header_size = (css->header_size_dw - css->modulus_size_dw - + css->key_size_dw - css->exponent_size_dw) * sizeof(u32); + + if (guc_fw->header_size != sizeof(struct guc_css_header)) { + DRM_ERROR("CSS header definition mismatch\n"); + goto fail; + } + + /* then, uCode */ + guc_fw->ucode_offset = guc_fw->header_offset + guc_fw->header_size; + guc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32); + + /* now RSA */ + if (css->key_size_dw != UOS_RSA_SCRATCH_MAX_COUNT) { + DRM_ERROR("RSA key size is bad\n"); + goto fail; + } + guc_fw->rsa_offset = guc_fw->ucode_offset + guc_fw->ucode_size; + guc_fw->rsa_size = css->key_size_dw * sizeof(u32); + + /* At least, it should have header, uCode and RSA. Size of all three. */ + size = guc_fw->header_size + guc_fw->ucode_size + guc_fw->rsa_size; + if (fw->size < size) { + DRM_ERROR("Missing firmware components\n"); + goto fail; + } + + /* Header and uCode will be loaded to WOPCM. Size of the two. */ + size = guc_fw->header_size + guc_fw->ucode_size; + + /* Top 32k of WOPCM is reserved (8K stack + 24k RC6 context). */ + if (size > GUC_WOPCM_SIZE_VALUE - 0x8000) { + DRM_ERROR("Firmware is too large to fit in WOPCM\n"); + goto fail; + } + + /* + * The GuC firmware image has the version number embedded at a well-known + * offset within the firmware blob; note that major / minor version are + * TWO bytes each (i.e. u16), although all pointers and offsets are defined + * in terms of bytes (u8). + */ + guc_fw->guc_fw_major_found = css->guc_sw_version >> 16; + guc_fw->guc_fw_minor_found = css->guc_sw_version & 0xFFFF; + + if (guc_fw->guc_fw_major_found != guc_fw->guc_fw_major_wanted || + guc_fw->guc_fw_minor_found < guc_fw->guc_fw_minor_wanted) { + DRM_ERROR("GuC firmware version %d.%d, required %d.%d\n", + guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found, + guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted); + err = -ENOEXEC; + goto fail; + } + + DRM_DEBUG_DRIVER("firmware version %d.%d OK (minimum %d.%d)\n", + guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found, + guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted); + + mutex_lock(&dev->struct_mutex); + obj = i915_gem_object_create_from_data(dev, fw->data, fw->size); + mutex_unlock(&dev->struct_mutex); + if (IS_ERR_OR_NULL(obj)) { + err = obj ? PTR_ERR(obj) : -ENOMEM; + goto fail; + } + + guc_fw->guc_fw_obj = obj; + guc_fw->guc_fw_size = fw->size; + + DRM_DEBUG_DRIVER("GuC fw fetch status SUCCESS, obj %p\n", + guc_fw->guc_fw_obj); + + release_firmware(fw); + guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_SUCCESS; + return; + +fail: + DRM_DEBUG_DRIVER("GuC fw fetch status FAIL; err %d, fw %p, obj %p\n", + err, fw, guc_fw->guc_fw_obj); + DRM_ERROR("Failed to fetch GuC firmware from %s (error %d)\n", + guc_fw->guc_fw_path, err); + + obj = guc_fw->guc_fw_obj; + if (obj) + drm_gem_object_unreference(&obj->base); + guc_fw->guc_fw_obj = NULL; + + release_firmware(fw); /* OK even if fw is NULL */ + guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_FAIL; +} + +/** + * intel_guc_ucode_init() - define parameters and fetch firmware + * @dev: drm device + * + * Called early during driver load, but after GEM is initialised. + * + * The firmware will be transferred to the GuC's memory later, + * when intel_guc_ucode_load() is called. + */ +void intel_guc_ucode_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; + const char *fw_path; + + if (!HAS_GUC_SCHED(dev)) + i915.enable_guc_submission = false; + + if (!HAS_GUC_UCODE(dev)) { + fw_path = NULL; + } else if (IS_SKYLAKE(dev)) { + fw_path = I915_SKL_GUC_UCODE; + guc_fw->guc_fw_major_wanted = 4; + guc_fw->guc_fw_minor_wanted = 3; + } else { + i915.enable_guc_submission = false; + fw_path = ""; /* unknown device */ + } + + if (!i915.enable_guc_submission) + return; + + guc_fw->guc_dev = dev; + guc_fw->guc_fw_path = fw_path; + guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE; + guc_fw->guc_fw_load_status = GUC_FIRMWARE_NONE; + + if (fw_path == NULL) + return; + + if (*fw_path == '\0') { + DRM_ERROR("No GuC firmware known for this platform\n"); + guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_FAIL; + return; + } + + guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_PENDING; + DRM_DEBUG_DRIVER("GuC firmware pending, path %s\n", fw_path); + guc_fw_fetch(dev, guc_fw); + /* status must now be FAIL or SUCCESS */ +} + +/** + * intel_guc_ucode_fini() - clean up all allocated resources + * @dev: drm device + */ +void intel_guc_ucode_fini(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; + + direct_interrupts_to_host(dev_priv); + i915_guc_submission_fini(dev); + + mutex_lock(&dev->struct_mutex); + if (guc_fw->guc_fw_obj) + drm_gem_object_unreference(&guc_fw->guc_fw_obj->base); + guc_fw->guc_fw_obj = NULL; + mutex_unlock(&dev->struct_mutex); + + guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE; +} diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index e97731aab6dc..00d065fee506 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -113,20 +113,22 @@ static u32 hsw_infoframe_enable(enum hdmi_infoframe_type type) } } -static u32 hsw_infoframe_data_reg(enum hdmi_infoframe_type type, - enum transcoder cpu_transcoder, - struct drm_i915_private *dev_priv) +static i915_reg_t +hsw_dip_data_reg(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder, + enum hdmi_infoframe_type type, + int i) { switch (type) { case HDMI_INFOFRAME_TYPE_AVI: - return HSW_TVIDEO_DIP_AVI_DATA(cpu_transcoder); + return HSW_TVIDEO_DIP_AVI_DATA(cpu_transcoder, i); case HDMI_INFOFRAME_TYPE_SPD: - return HSW_TVIDEO_DIP_SPD_DATA(cpu_transcoder); + return HSW_TVIDEO_DIP_SPD_DATA(cpu_transcoder, i); case HDMI_INFOFRAME_TYPE_VENDOR: - return HSW_TVIDEO_DIP_VS_DATA(cpu_transcoder); + return HSW_TVIDEO_DIP_VS_DATA(cpu_transcoder, i); default: DRM_DEBUG_DRIVER("unknown info frame type %d\n", type); - return 0; + return INVALID_MMIO_REG; } } @@ -167,17 +169,21 @@ static void g4x_write_infoframe(struct drm_encoder *encoder, POSTING_READ(VIDEO_DIP_CTL); } -static bool g4x_infoframe_enabled(struct drm_encoder *encoder) +static bool g4x_infoframe_enabled(struct drm_encoder *encoder, + const struct intel_crtc_state *pipe_config) { - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); u32 val = I915_READ(VIDEO_DIP_CTL); - if (VIDEO_DIP_PORT(intel_dig_port->port) == (val & VIDEO_DIP_PORT_MASK)) - return val & VIDEO_DIP_ENABLE; + if ((val & VIDEO_DIP_ENABLE) == 0) + return false; - return false; + if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port)) + return false; + + return val & (VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_SPD); } static void ibx_write_infoframe(struct drm_encoder *encoder, @@ -188,8 +194,9 @@ static void ibx_write_infoframe(struct drm_encoder *encoder, struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); - int i, reg = TVIDEO_DIP_CTL(intel_crtc->pipe); + i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); + int i; WARN(!(val & VIDEO_DIP_ENABLE), "Writing DIP with CTL reg disabled\n"); @@ -218,19 +225,24 @@ static void ibx_write_infoframe(struct drm_encoder *encoder, POSTING_READ(reg); } -static bool ibx_infoframe_enabled(struct drm_encoder *encoder) +static bool ibx_infoframe_enabled(struct drm_encoder *encoder, + const struct intel_crtc_state *pipe_config) { - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); - int reg = TVIDEO_DIP_CTL(intel_crtc->pipe); + enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe; + i915_reg_t reg = TVIDEO_DIP_CTL(pipe); u32 val = I915_READ(reg); - if (VIDEO_DIP_PORT(intel_dig_port->port) == (val & VIDEO_DIP_PORT_MASK)) - return val & VIDEO_DIP_ENABLE; + if ((val & VIDEO_DIP_ENABLE) == 0) + return false; - return false; + if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port)) + return false; + + return val & (VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); } static void cpt_write_infoframe(struct drm_encoder *encoder, @@ -241,8 +253,9 @@ static void cpt_write_infoframe(struct drm_encoder *encoder, struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); - int i, reg = TVIDEO_DIP_CTL(intel_crtc->pipe); + i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); + int i; WARN(!(val & VIDEO_DIP_ENABLE), "Writing DIP with CTL reg disabled\n"); @@ -274,15 +287,19 @@ static void cpt_write_infoframe(struct drm_encoder *encoder, POSTING_READ(reg); } -static bool cpt_infoframe_enabled(struct drm_encoder *encoder) +static bool cpt_infoframe_enabled(struct drm_encoder *encoder, + const struct intel_crtc_state *pipe_config) { - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); - int reg = TVIDEO_DIP_CTL(intel_crtc->pipe); - u32 val = I915_READ(reg); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); + enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe; + u32 val = I915_READ(TVIDEO_DIP_CTL(pipe)); + + if ((val & VIDEO_DIP_ENABLE) == 0) + return false; - return val & VIDEO_DIP_ENABLE; + return val & (VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); } static void vlv_write_infoframe(struct drm_encoder *encoder, @@ -293,8 +310,9 @@ static void vlv_write_infoframe(struct drm_encoder *encoder, struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); - int i, reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); + i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); + int i; WARN(!(val & VIDEO_DIP_ENABLE), "Writing DIP with CTL reg disabled\n"); @@ -323,19 +341,23 @@ static void vlv_write_infoframe(struct drm_encoder *encoder, POSTING_READ(reg); } -static bool vlv_infoframe_enabled(struct drm_encoder *encoder) +static bool vlv_infoframe_enabled(struct drm_encoder *encoder, + const struct intel_crtc_state *pipe_config) { - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); - int reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); - u32 val = I915_READ(reg); + enum pipe pipe = to_intel_crtc(pipe_config->base.crtc)->pipe; + u32 val = I915_READ(VLV_TVIDEO_DIP_CTL(pipe)); - if (VIDEO_DIP_PORT(intel_dig_port->port) == (val & VIDEO_DIP_PORT_MASK)) - return val & VIDEO_DIP_ENABLE; + if ((val & VIDEO_DIP_ENABLE) == 0) + return false; - return false; + if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port)) + return false; + + return val & (VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); } static void hsw_write_infoframe(struct drm_encoder *encoder, @@ -346,15 +368,14 @@ static void hsw_write_infoframe(struct drm_encoder *encoder, struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); - u32 ctl_reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder); - u32 data_reg; + enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; + i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder); + i915_reg_t data_reg; int i; u32 val = I915_READ(ctl_reg); - data_reg = hsw_infoframe_data_reg(type, - intel_crtc->config->cpu_transcoder, - dev_priv); - if (data_reg == 0) + data_reg = hsw_dip_data_reg(dev_priv, cpu_transcoder, type, 0); + if (i915_mmio_reg_valid(data_reg)) return; val &= ~hsw_infoframe_enable(type); @@ -362,12 +383,14 @@ static void hsw_write_infoframe(struct drm_encoder *encoder, mmiowb(); for (i = 0; i < len; i += 4) { - I915_WRITE(data_reg + i, *data); + I915_WRITE(hsw_dip_data_reg(dev_priv, cpu_transcoder, + type, i >> 2), *data); data++; } /* Write every possible data byte to force correct ECC calculation. */ for (; i < VIDEO_DIP_DATA_SIZE; i += 4) - I915_WRITE(data_reg + i, 0); + I915_WRITE(hsw_dip_data_reg(dev_priv, cpu_transcoder, + type, i >> 2), 0); mmiowb(); val |= hsw_infoframe_enable(type); @@ -375,16 +398,15 @@ static void hsw_write_infoframe(struct drm_encoder *encoder, POSTING_READ(ctl_reg); } -static bool hsw_infoframe_enabled(struct drm_encoder *encoder) +static bool hsw_infoframe_enabled(struct drm_encoder *encoder, + const struct intel_crtc_state *pipe_config) { - struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); - u32 ctl_reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder); - u32 val = I915_READ(ctl_reg); + struct drm_i915_private *dev_priv = to_i915(encoder->dev); + u32 val = I915_READ(HSW_TVIDEO_DIP_CTL(pipe_config->cpu_transcoder)); - return val & (VIDEO_DIP_ENABLE_AVI_HSW | VIDEO_DIP_ENABLE_SPD_HSW | - VIDEO_DIP_ENABLE_VS_HSW); + return val & (VIDEO_DIP_ENABLE_VSC_HSW | VIDEO_DIP_ENABLE_AVI_HSW | + VIDEO_DIP_ENABLE_GCP_HSW | VIDEO_DIP_ENABLE_VS_HSW | + VIDEO_DIP_ENABLE_GMP_HSW | VIDEO_DIP_ENABLE_SPD_HSW); } /* @@ -427,16 +449,13 @@ static void intel_write_infoframe(struct drm_encoder *encoder, } static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); union hdmi_infoframe frame; int ret; - /* Set user selected PAR to incoming mode's member */ - adjusted_mode->picture_aspect_ratio = intel_hdmi->aspect_ratio; - ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, adjusted_mode); if (ret < 0) { @@ -474,7 +493,7 @@ static void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder) static void intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { union hdmi_infoframe frame; int ret; @@ -489,12 +508,12 @@ intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder, static void g4x_set_infoframes(struct drm_encoder *encoder, bool enable, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; - u32 reg = VIDEO_DIP_CTL; + i915_reg_t reg = VIDEO_DIP_CTL; u32 val = I915_READ(reg); u32 port = VIDEO_DIP_PORT(intel_dig_port->port); @@ -514,7 +533,13 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; - val &= ~VIDEO_DIP_ENABLE; + if (port != (val & VIDEO_DIP_PORT_MASK)) { + DRM_DEBUG_KMS("video DIP still enabled on port %c\n", + (val & VIDEO_DIP_PORT_MASK) >> 29); + return; + } + val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_SPD); I915_WRITE(reg, val); POSTING_READ(reg); return; @@ -522,16 +547,17 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, if (port != (val & VIDEO_DIP_PORT_MASK)) { if (val & VIDEO_DIP_ENABLE) { - val &= ~VIDEO_DIP_ENABLE; - I915_WRITE(reg, val); - POSTING_READ(reg); + DRM_DEBUG_KMS("video DIP already enabled on port %c\n", + (val & VIDEO_DIP_PORT_MASK) >> 29); + return; } val &= ~VIDEO_DIP_PORT_MASK; val |= port; } val |= VIDEO_DIP_ENABLE; - val &= ~VIDEO_DIP_ENABLE_VENDOR; + val &= ~(VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_SPD); I915_WRITE(reg, val); POSTING_READ(reg); @@ -541,15 +567,107 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, intel_hdmi_set_hdmi_infoframe(encoder, adjusted_mode); } +static bool hdmi_sink_is_deep_color(struct drm_encoder *encoder) +{ + struct drm_device *dev = encoder->dev; + struct drm_connector *connector; + + WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + + /* + * HDMI cloning is only supported on g4x which doesn't + * support deep color or GCP infoframes anyway so no + * need to worry about multiple HDMI sinks here. + */ + list_for_each_entry(connector, &dev->mode_config.connector_list, head) + if (connector->encoder == encoder) + return connector->display_info.bpc > 8; + + return false; +} + +/* + * Determine if default_phase=1 can be indicated in the GCP infoframe. + * + * From HDMI specification 1.4a: + * - The first pixel of each Video Data Period shall always have a pixel packing phase of 0 + * - The first pixel following each Video Data Period shall have a pixel packing phase of 0 + * - The PP bits shall be constant for all GCPs and will be equal to the last packing phase + * - The first pixel following every transition of HSYNC or VSYNC shall have a pixel packing + * phase of 0 + */ +static bool gcp_default_phase_possible(int pipe_bpp, + const struct drm_display_mode *mode) +{ + unsigned int pixels_per_group; + + switch (pipe_bpp) { + case 30: + /* 4 pixels in 5 clocks */ + pixels_per_group = 4; + break; + case 36: + /* 2 pixels in 3 clocks */ + pixels_per_group = 2; + break; + case 48: + /* 1 pixel in 2 clocks */ + pixels_per_group = 1; + break; + default: + /* phase information not relevant for 8bpc */ + return false; + } + + return mode->crtc_hdisplay % pixels_per_group == 0 && + mode->crtc_htotal % pixels_per_group == 0 && + mode->crtc_hblank_start % pixels_per_group == 0 && + mode->crtc_hblank_end % pixels_per_group == 0 && + mode->crtc_hsync_start % pixels_per_group == 0 && + mode->crtc_hsync_end % pixels_per_group == 0 && + ((mode->flags & DRM_MODE_FLAG_INTERLACE) == 0 || + mode->crtc_htotal/2 % pixels_per_group == 0); +} + +static bool intel_hdmi_set_gcp_infoframe(struct drm_encoder *encoder) +{ + struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); + i915_reg_t reg; + u32 val = 0; + + if (HAS_DDI(dev_priv)) + reg = HSW_TVIDEO_DIP_GCP(crtc->config->cpu_transcoder); + else if (IS_VALLEYVIEW(dev_priv)) + reg = VLV_TVIDEO_DIP_GCP(crtc->pipe); + else if (HAS_PCH_SPLIT(dev_priv->dev)) + reg = TVIDEO_DIP_GCP(crtc->pipe); + else + return false; + + /* Indicate color depth whenever the sink supports deep color */ + if (hdmi_sink_is_deep_color(encoder)) + val |= GCP_COLOR_INDICATION; + + /* Enable default_phase whenever the display mode is suitably aligned */ + if (gcp_default_phase_possible(crtc->config->pipe_bpp, + &crtc->config->base.adjusted_mode)) + val |= GCP_DEFAULT_PHASE_ENABLE; + + I915_WRITE(reg, val); + + return val != 0; +} + static void ibx_set_infoframes(struct drm_encoder *encoder, bool enable, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; - u32 reg = TVIDEO_DIP_CTL(intel_crtc->pipe); + i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); u32 port = VIDEO_DIP_PORT(intel_dig_port->port); @@ -561,25 +679,29 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; - val &= ~VIDEO_DIP_ENABLE; + val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); I915_WRITE(reg, val); POSTING_READ(reg); return; } if (port != (val & VIDEO_DIP_PORT_MASK)) { - if (val & VIDEO_DIP_ENABLE) { - val &= ~VIDEO_DIP_ENABLE; - I915_WRITE(reg, val); - POSTING_READ(reg); - } + WARN(val & VIDEO_DIP_ENABLE, + "DIP already enabled on port %c\n", + (val & VIDEO_DIP_PORT_MASK) >> 29); val &= ~VIDEO_DIP_PORT_MASK; val |= port; } val |= VIDEO_DIP_ENABLE; - val &= ~(VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | - VIDEO_DIP_ENABLE_GCP); + val &= ~(VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); + + if (intel_hdmi_set_gcp_infoframe(encoder)) + val |= VIDEO_DIP_ENABLE_GCP; I915_WRITE(reg, val); POSTING_READ(reg); @@ -591,12 +713,12 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, static void cpt_set_infoframes(struct drm_encoder *encoder, bool enable, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); - u32 reg = TVIDEO_DIP_CTL(intel_crtc->pipe); + i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); assert_hdmi_port_disabled(intel_hdmi); @@ -607,7 +729,9 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; - val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI); + val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); I915_WRITE(reg, val); POSTING_READ(reg); return; @@ -616,7 +740,10 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, /* Set both together, unset both together: see the spec. */ val |= VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI; val &= ~(VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | - VIDEO_DIP_ENABLE_GCP); + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); + + if (intel_hdmi_set_gcp_infoframe(encoder)) + val |= VIDEO_DIP_ENABLE_GCP; I915_WRITE(reg, val); POSTING_READ(reg); @@ -628,13 +755,13 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, static void vlv_set_infoframes(struct drm_encoder *encoder, bool enable, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); - u32 reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); + i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); u32 port = VIDEO_DIP_PORT(intel_dig_port->port); @@ -646,25 +773,29 @@ static void vlv_set_infoframes(struct drm_encoder *encoder, if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; - val &= ~VIDEO_DIP_ENABLE; + val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); I915_WRITE(reg, val); POSTING_READ(reg); return; } if (port != (val & VIDEO_DIP_PORT_MASK)) { - if (val & VIDEO_DIP_ENABLE) { - val &= ~VIDEO_DIP_ENABLE; - I915_WRITE(reg, val); - POSTING_READ(reg); - } + WARN(val & VIDEO_DIP_ENABLE, + "DIP already enabled on port %c\n", + (val & VIDEO_DIP_PORT_MASK) >> 29); val &= ~VIDEO_DIP_PORT_MASK; val |= port; } val |= VIDEO_DIP_ENABLE; - val &= ~(VIDEO_DIP_ENABLE_AVI | VIDEO_DIP_ENABLE_VENDOR | - VIDEO_DIP_ENABLE_GAMUT | VIDEO_DIP_ENABLE_GCP); + val &= ~(VIDEO_DIP_ENABLE_AVI | + VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | + VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_ENABLE_GCP); + + if (intel_hdmi_set_gcp_infoframe(encoder)) + val |= VIDEO_DIP_ENABLE_GCP; I915_WRITE(reg, val); POSTING_READ(reg); @@ -676,24 +807,28 @@ static void vlv_set_infoframes(struct drm_encoder *encoder, static void hsw_set_infoframes(struct drm_encoder *encoder, bool enable, - struct drm_display_mode *adjusted_mode) + const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); - u32 reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder); + i915_reg_t reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder); u32 val = I915_READ(reg); assert_hdmi_port_disabled(intel_hdmi); + val &= ~(VIDEO_DIP_ENABLE_VSC_HSW | VIDEO_DIP_ENABLE_AVI_HSW | + VIDEO_DIP_ENABLE_GCP_HSW | VIDEO_DIP_ENABLE_VS_HSW | + VIDEO_DIP_ENABLE_GMP_HSW | VIDEO_DIP_ENABLE_SPD_HSW); + if (!enable) { - I915_WRITE(reg, 0); + I915_WRITE(reg, val); POSTING_READ(reg); return; } - val &= ~(VIDEO_DIP_ENABLE_VSC_HSW | VIDEO_DIP_ENABLE_GCP_HSW | - VIDEO_DIP_ENABLE_VS_HSW | VIDEO_DIP_ENABLE_GMP_HSW); + if (intel_hdmi_set_gcp_infoframe(encoder)) + val |= VIDEO_DIP_ENABLE_GCP_HSW; I915_WRITE(reg, val); POSTING_READ(reg); @@ -709,12 +844,12 @@ static void intel_hdmi_prepare(struct intel_encoder *encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); - struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; u32 hdmi_val; hdmi_val = SDVO_ENCODING_HDMI; - if (!HAS_PCH_SPLIT(dev)) - hdmi_val |= intel_hdmi->color_range; + if (!HAS_PCH_SPLIT(dev) && crtc->config->limited_color_range) + hdmi_val |= HDMI_COLOR_RANGE_16_235; if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) hdmi_val |= SDVO_VSYNC_ACTIVE_HIGH; if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) @@ -791,7 +926,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, if (tmp & HDMI_MODE_SELECT_HDMI) pipe_config->has_hdmi_sink = true; - if (intel_hdmi->infoframe_enabled(&encoder->base)) + if (intel_hdmi->infoframe_enabled(&encoder->base, pipe_config)) pipe_config->has_infoframe = true; if (tmp & SDVO_AUDIO_ENABLE) @@ -808,58 +943,146 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, else dotclock = pipe_config->port_clock; + if (pipe_config->pixel_multiplier) + dotclock /= pipe_config->pixel_multiplier; + if (HAS_PCH_SPLIT(dev_priv->dev)) ironlake_check_encoder_dotclock(pipe_config, dotclock); pipe_config->base.adjusted_mode.crtc_clock = dotclock; } -static void intel_enable_hdmi(struct intel_encoder *encoder) +static void intel_enable_hdmi_audio(struct intel_encoder *encoder) +{ + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + + WARN_ON(!crtc->config->has_hdmi_sink); + DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n", + pipe_name(crtc->pipe)); + intel_audio_codec_enable(encoder); +} + +static void g4x_enable_hdmi(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); u32 temp; - u32 enable_bits = SDVO_ENABLE; - if (intel_crtc->config->has_audio) - enable_bits |= SDVO_AUDIO_ENABLE; + temp = I915_READ(intel_hdmi->hdmi_reg); + + temp |= SDVO_ENABLE; + if (crtc->config->has_audio) + temp |= SDVO_AUDIO_ENABLE; + + I915_WRITE(intel_hdmi->hdmi_reg, temp); + POSTING_READ(intel_hdmi->hdmi_reg); + + if (crtc->config->has_audio) + intel_enable_hdmi_audio(encoder); +} + +static void ibx_enable_hdmi(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); + u32 temp; temp = I915_READ(intel_hdmi->hdmi_reg); - /* HW workaround for IBX, we need to move the port to transcoder A - * before disabling it, so restore the transcoder select bit here. */ - if (HAS_PCH_IBX(dev)) - enable_bits |= SDVO_PIPE_SEL(intel_crtc->pipe); + temp |= SDVO_ENABLE; + if (crtc->config->has_audio) + temp |= SDVO_AUDIO_ENABLE; - /* HW workaround, need to toggle enable bit off and on for 12bpc, but - * we do this anyway which shows more stable in testing. + /* + * HW workaround, need to write this twice for issue + * that may result in first write getting masked. */ - if (HAS_PCH_SPLIT(dev)) { + I915_WRITE(intel_hdmi->hdmi_reg, temp); + POSTING_READ(intel_hdmi->hdmi_reg); + I915_WRITE(intel_hdmi->hdmi_reg, temp); + POSTING_READ(intel_hdmi->hdmi_reg); + + /* + * HW workaround, need to toggle enable bit off and on + * for 12bpc with pixel repeat. + * + * FIXME: BSpec says this should be done at the end of + * of the modeset sequence, so not sure if this isn't too soon. + */ + if (crtc->config->pipe_bpp > 24 && + crtc->config->pixel_multiplier > 1) { I915_WRITE(intel_hdmi->hdmi_reg, temp & ~SDVO_ENABLE); POSTING_READ(intel_hdmi->hdmi_reg); + + /* + * HW workaround, need to write this twice for issue + * that may result in first write getting masked. + */ + I915_WRITE(intel_hdmi->hdmi_reg, temp); + POSTING_READ(intel_hdmi->hdmi_reg); + I915_WRITE(intel_hdmi->hdmi_reg, temp); + POSTING_READ(intel_hdmi->hdmi_reg); } - temp |= enable_bits; + if (crtc->config->has_audio) + intel_enable_hdmi_audio(encoder); +} + +static void cpt_enable_hdmi(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); + enum pipe pipe = crtc->pipe; + u32 temp; + + temp = I915_READ(intel_hdmi->hdmi_reg); + + temp |= SDVO_ENABLE; + if (crtc->config->has_audio) + temp |= SDVO_AUDIO_ENABLE; + + /* + * WaEnableHDMI8bpcBefore12bpc:snb,ivb + * + * The procedure for 12bpc is as follows: + * 1. disable HDMI clock gating + * 2. enable HDMI with 8bpc + * 3. enable HDMI with 12bpc + * 4. enable HDMI clock gating + */ + + if (crtc->config->pipe_bpp > 24) { + I915_WRITE(TRANS_CHICKEN1(pipe), + I915_READ(TRANS_CHICKEN1(pipe)) | + TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE); + + temp &= ~SDVO_COLOR_FORMAT_MASK; + temp |= SDVO_COLOR_FORMAT_8bpc; + } I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - /* HW workaround, need to write this twice for issue that may result - * in first write getting masked. - */ - if (HAS_PCH_SPLIT(dev)) { + if (crtc->config->pipe_bpp > 24) { + temp &= ~SDVO_COLOR_FORMAT_MASK; + temp |= HDMI_COLOR_FORMAT_12bpc; + I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - } - if (intel_crtc->config->has_audio) { - WARN_ON(!intel_crtc->config->has_hdmi_sink); - DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n", - pipe_name(intel_crtc->pipe)); - intel_audio_codec_enable(encoder); + I915_WRITE(TRANS_CHICKEN1(pipe), + I915_READ(TRANS_CHICKEN1(pipe)) & + ~TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE); } + + if (crtc->config->has_audio) + intel_enable_hdmi_audio(encoder); } static void vlv_enable_hdmi(struct intel_encoder *encoder) @@ -886,6 +1109,13 @@ static void intel_disable_hdmi(struct intel_encoder *encoder) * matching DP port to be enabled on transcoder A. */ if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B) { + /* + * We get CPU/PCH FIFO underruns on the other pipe when + * doing the workaround. Sweep them under the rug. + */ + intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false); + intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); + temp &= ~SDVO_PIPE_B_SELECT; temp |= SDVO_ENABLE; /* @@ -900,7 +1130,13 @@ static void intel_disable_hdmi(struct intel_encoder *encoder) temp &= ~SDVO_ENABLE; I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); + + intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A); + intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); + intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } + + intel_hdmi->set_infoframes(&encoder->base, false, NULL); } static void g4x_disable_hdmi(struct intel_encoder *encoder) @@ -926,7 +1162,7 @@ static void pch_post_disable_hdmi(struct intel_encoder *encoder) intel_disable_hdmi(encoder); } -static int hdmi_portclock_limit(struct intel_hdmi *hdmi, bool respect_dvi_limit) +static int hdmi_port_clock_limit(struct intel_hdmi *hdmi, bool respect_dvi_limit) { struct drm_device *dev = intel_hdmi_to_dev(hdmi); @@ -939,24 +1175,51 @@ static int hdmi_portclock_limit(struct intel_hdmi *hdmi, bool respect_dvi_limit) } static enum drm_mode_status +hdmi_port_clock_valid(struct intel_hdmi *hdmi, + int clock, bool respect_dvi_limit) +{ + struct drm_device *dev = intel_hdmi_to_dev(hdmi); + + if (clock < 25000) + return MODE_CLOCK_LOW; + if (clock > hdmi_port_clock_limit(hdmi, respect_dvi_limit)) + return MODE_CLOCK_HIGH; + + /* BXT DPLL can't generate 223-240 MHz */ + if (IS_BROXTON(dev) && clock > 223333 && clock < 240000) + return MODE_CLOCK_RANGE; + + /* CHV DPLL can't generate 216-240 MHz */ + if (IS_CHERRYVIEW(dev) && clock > 216000 && clock < 240000) + return MODE_CLOCK_RANGE; + + return MODE_OK; +} + +static enum drm_mode_status intel_hdmi_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { - int clock = mode->clock; + struct intel_hdmi *hdmi = intel_attached_hdmi(connector); + struct drm_device *dev = intel_hdmi_to_dev(hdmi); + enum drm_mode_status status; + int clock; + + if (mode->flags & DRM_MODE_FLAG_DBLSCAN) + return MODE_NO_DBLESCAN; + clock = mode->clock; if (mode->flags & DRM_MODE_FLAG_DBLCLK) clock *= 2; - if (clock > hdmi_portclock_limit(intel_attached_hdmi(connector), - true)) - return MODE_CLOCK_HIGH; - if (clock < 20000) - return MODE_CLOCK_LOW; + /* check if we can do 8bpc */ + status = hdmi_port_clock_valid(hdmi, clock, true); - if (mode->flags & DRM_MODE_FLAG_DBLSCAN) - return MODE_NO_DBLESCAN; + /* if we can't do 8bpc we may still be able to do 12bpc */ + if (!HAS_GMCH_DISPLAY(dev) && status != MODE_OK) + status = hdmi_port_clock_valid(hdmi, clock * 3 / 2, true); - return MODE_OK; + return status; } static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state) @@ -997,8 +1260,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); struct drm_device *dev = encoder->base.dev; struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; - int clock_12bpc = pipe_config->base.adjusted_mode.crtc_clock * 3 / 2; - int portclock_limit = hdmi_portclock_limit(intel_hdmi, false); + int clock_8bpc = pipe_config->base.adjusted_mode.crtc_clock; + int clock_12bpc = clock_8bpc * 3 / 2; int desired_bpp; pipe_config->has_hdmi_sink = intel_hdmi->has_hdmi_sink; @@ -1008,20 +1271,20 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, if (intel_hdmi->color_range_auto) { /* See CEA-861-E - 5.1 Default Encoding Parameters */ - if (pipe_config->has_hdmi_sink && - drm_match_cea_mode(adjusted_mode) > 1) - intel_hdmi->color_range = HDMI_COLOR_RANGE_16_235; - else - intel_hdmi->color_range = 0; + pipe_config->limited_color_range = + pipe_config->has_hdmi_sink && + drm_match_cea_mode(adjusted_mode) > 1; + } else { + pipe_config->limited_color_range = + intel_hdmi->limited_color_range; } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) { pipe_config->pixel_multiplier = 2; + clock_8bpc *= 2; + clock_12bpc *= 2; } - if (intel_hdmi->color_range) - pipe_config->limited_color_range = true; - if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev)) pipe_config->has_pch_encoder = true; @@ -1035,9 +1298,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, * within limits. */ if (pipe_config->pipe_bpp > 8*3 && pipe_config->has_hdmi_sink && - clock_12bpc <= portclock_limit && - hdmi_12bpc_possible(pipe_config) && - 0 /* FIXME 12bpc support totally broken */) { + hdmi_port_clock_valid(intel_hdmi, clock_12bpc, false) == MODE_OK && + hdmi_12bpc_possible(pipe_config)) { DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n"); desired_bpp = 12*3; @@ -1046,6 +1308,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, } else { DRM_DEBUG_KMS("picking bpc to 8 for HDMI output\n"); desired_bpp = 8*3; + + pipe_config->port_clock = clock_8bpc; } if (!pipe_config->bw_constrained) { @@ -1053,11 +1317,15 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, pipe_config->pipe_bpp = desired_bpp; } - if (adjusted_mode->crtc_clock > portclock_limit) { - DRM_DEBUG_KMS("too high HDMI clock, rejecting mode\n"); + if (hdmi_port_clock_valid(intel_hdmi, pipe_config->port_clock, + false) != MODE_OK) { + DRM_DEBUG_KMS("unsupported HDMI clock, rejecting mode\n"); return false; } + /* Set user selected PAR to incoming mode's member */ + adjusted_mode->picture_aspect_ratio = intel_hdmi->aspect_ratio; + return true; } @@ -1075,24 +1343,22 @@ intel_hdmi_unset_edid(struct drm_connector *connector) } static bool -intel_hdmi_set_edid(struct drm_connector *connector) +intel_hdmi_set_edid(struct drm_connector *connector, bool force) { struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); - struct intel_encoder *intel_encoder = - &hdmi_to_dig_port(intel_hdmi)->base; - enum intel_display_power_domain power_domain; - struct edid *edid; + struct edid *edid = NULL; bool connected = false; - power_domain = intel_display_port_power_domain(intel_encoder); - intel_display_power_get(dev_priv, power_domain); + if (force) { + intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); - edid = drm_get_edid(connector, - intel_gmbus_get_adapter(dev_priv, - intel_hdmi->ddc_bus)); + edid = drm_get_edid(connector, + intel_gmbus_get_adapter(dev_priv, + intel_hdmi->ddc_bus)); - intel_display_power_put(dev_priv, power_domain); + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + } to_intel_connector(connector)->detect_edid = edid; if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) { @@ -1118,13 +1384,28 @@ static enum drm_connector_status intel_hdmi_detect(struct drm_connector *connector, bool force) { enum drm_connector_status status; + struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + struct drm_i915_private *dev_priv = to_i915(connector->dev); + bool live_status = false; + unsigned int retry = 3; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); + intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + + while (!live_status && --retry) { + live_status = intel_digital_port_connected(dev_priv, + hdmi_to_dig_port(intel_hdmi)); + mdelay(10); + } + + if (!live_status) + DRM_DEBUG_KMS("Live status not up!"); + intel_hdmi_unset_edid(connector); - if (intel_hdmi_set_edid(connector)) { + if (intel_hdmi_set_edid(connector, live_status)) { struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI; @@ -1132,6 +1413,8 @@ intel_hdmi_detect(struct drm_connector *connector, bool force) } else status = connector_status_disconnected; + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + return status; } @@ -1148,7 +1431,7 @@ intel_hdmi_force(struct drm_connector *connector) if (connector->status != connector_status_connected) return; - intel_hdmi_set_edid(connector); + intel_hdmi_set_edid(connector, true); hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI; } @@ -1214,7 +1497,7 @@ intel_hdmi_set_property(struct drm_connector *connector, if (property == dev_priv->broadcast_rgb_property) { bool old_auto = intel_hdmi->color_range_auto; - uint32_t old_range = intel_hdmi->color_range; + bool old_range = intel_hdmi->limited_color_range; switch (val) { case INTEL_BROADCAST_RGB_AUTO: @@ -1222,18 +1505,18 @@ intel_hdmi_set_property(struct drm_connector *connector, break; case INTEL_BROADCAST_RGB_FULL: intel_hdmi->color_range_auto = false; - intel_hdmi->color_range = 0; + intel_hdmi->limited_color_range = false; break; case INTEL_BROADCAST_RGB_LIMITED: intel_hdmi->color_range_auto = false; - intel_hdmi->color_range = HDMI_COLOR_RANGE_16_235; + intel_hdmi->limited_color_range = true; break; default: return -EINVAL; } if (old_auto == intel_hdmi->color_range_auto && - old_range == intel_hdmi->color_range) + old_range == intel_hdmi->limited_color_range) return 0; goto done; @@ -1269,8 +1552,7 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder) { struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - struct drm_display_mode *adjusted_mode = - &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; intel_hdmi_prepare(encoder); @@ -1287,8 +1569,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - struct drm_display_mode *adjusted_mode = - &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; enum dpio_channel port = vlv_dport_to_channel(dport); int pipe = intel_crtc->pipe; u32 val; @@ -1323,7 +1604,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder) intel_crtc->config->has_hdmi_sink, adjusted_mode); - intel_enable_hdmi(encoder); + g4x_enable_hdmi(encoder); vlv_wait_port_ready(dev_priv, dport, 0x0); } @@ -1361,6 +1642,50 @@ static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder) mutex_unlock(&dev_priv->sb_lock); } +static void chv_data_lane_soft_reset(struct intel_encoder *encoder, + bool reset) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + enum pipe pipe = crtc->pipe; + uint32_t val; + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + if (reset) + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + else + val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); + + if (crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + if (reset) + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + else + val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + } + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + if (reset) + val &= ~DPIO_PCS_CLK_SOFT_RESET; + else + val |= DPIO_PCS_CLK_SOFT_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); + + if (crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + if (reset) + val &= ~DPIO_PCS_CLK_SOFT_RESET; + else + val |= DPIO_PCS_CLK_SOFT_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + } +} + static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder) { struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); @@ -1374,8 +1699,21 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder) intel_hdmi_prepare(encoder); + /* + * Must trick the second common lane into life. + * Otherwise we can't even access the PLL. + */ + if (ch == DPIO_CH0 && pipe == PIPE_B) + dport->release_cl2_override = + !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); + + chv_phy_powergate_lanes(encoder, true, 0x0); + mutex_lock(&dev_priv->sb_lock); + /* Assert data lane reset */ + chv_data_lane_soft_reset(encoder, true); + /* program left/right clock distribution */ if (pipe != PIPE_B) { val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); @@ -1427,6 +1765,39 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder) mutex_unlock(&dev_priv->sb_lock); } +static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe; + u32 val; + + mutex_lock(&dev_priv->sb_lock); + + /* disable left/right clock distribution */ + if (pipe != PIPE_B) { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); + val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); + } else { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); + val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); + } + + mutex_unlock(&dev_priv->sb_lock); + + /* + * Leave the power down bit cleared for at least one + * lane so that chv_powergate_phy_ch() will power + * on something when the channel is otherwise unused. + * When the port is off and the override is removed + * the lanes power down anyway, so otherwise it doesn't + * really matter what the state of power down bits is + * after this. + */ + chv_phy_powergate_lanes(encoder, false, 0x0); +} + static void vlv_hdmi_post_disable(struct intel_encoder *encoder) { struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); @@ -1445,33 +1816,13 @@ static void vlv_hdmi_post_disable(struct intel_encoder *encoder) static void chv_hdmi_post_disable(struct intel_encoder *encoder) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel ch = vlv_dport_to_channel(dport); - enum pipe pipe = intel_crtc->pipe; - u32 val; mutex_lock(&dev_priv->sb_lock); - /* Propagate soft reset to data lane reset */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + /* Assert data lane reset */ + chv_data_lane_soft_reset(encoder, true); mutex_unlock(&dev_priv->sb_lock); } @@ -1484,8 +1835,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - struct drm_display_mode *adjusted_mode = - &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; enum dpio_channel ch = vlv_dport_to_channel(dport); int pipe = intel_crtc->pipe; int data, i, stagger; @@ -1502,23 +1852,6 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) val &= ~DPIO_LANEDESKEW_STRAP_OVRD; vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); - /* Deassert soft data lane reset*/ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); - val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); - val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); - /* Program Tx latency optimal setting */ for (i = 0; i < 4; i++) { /* Set the upar bit */ @@ -1561,6 +1894,9 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) DPIO_TX1_STAGGER_MULT(7) | DPIO_TX2_STAGGER_MULT(5)); + /* Deassert data lane reset */ + chv_data_lane_soft_reset(encoder, false); + /* Clear calc init */ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); @@ -1595,31 +1931,33 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) for (i = 0; i < 4; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); + val &= ~DPIO_SWING_MARGIN000_MASK; val |= 102 << DPIO_SWING_MARGIN000_SHIFT; + + /* + * Supposedly this value shouldn't matter when unique transition + * scale is disabled, but in fact it does matter. Let's just + * always program the same value and hope it's OK. + */ + val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); + val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); } - /* Disable unique transition scale */ + /* + * The document said it needs to set bit 27 for ch0 and bit 26 + * for ch1. Might be a typo in the doc. + * For now, for this unique transition scale selection, set bit + * 27 for ch0 and ch1. + */ for (i = 0; i < 4; i++) { val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); } - /* Additional steps for 1200mV-0dB */ -#if 0 - val = vlv_dpio_read(dev_priv, pipe, VLV_TX_DW3(ch)); - if (ch) - val |= DPIO_TX_UNIQ_TRANS_SCALE_CH1; - else - val |= DPIO_TX_UNIQ_TRANS_SCALE_CH0; - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(ch), val); - - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(ch), - vlv_dpio_read(dev_priv, pipe, VLV_TX_DW2(ch)) | - (0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT)); -#endif /* Start swing calculation */ val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; @@ -1629,20 +1967,21 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); - /* LRC Bypass */ - val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30); - val |= DPIO_LRC_BYPASS; - vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, val); - mutex_unlock(&dev_priv->sb_lock); intel_hdmi->set_infoframes(&encoder->base, intel_crtc->config->has_hdmi_sink, adjusted_mode); - intel_enable_hdmi(encoder); + g4x_enable_hdmi(encoder); vlv_wait_port_ready(dev_priv, dport, 0x0); + + /* Second common lane will stay alive on its own now */ + if (dport->release_cl2_override) { + chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); + dport->release_cl2_override = false; + } } static void intel_hdmi_destroy(struct drm_connector *connector) @@ -1653,7 +1992,7 @@ static void intel_hdmi_destroy(struct drm_connector *connector) } static const struct drm_connector_funcs intel_hdmi_connector_funcs = { - .dpms = intel_connector_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_hdmi_detect, .force = intel_hdmi_force, .fill_modes = drm_helper_probe_single_connector_modes, @@ -1675,15 +2014,6 @@ static const struct drm_encoder_funcs intel_hdmi_enc_funcs = { }; static void -intel_attach_aspect_ratio_property(struct drm_connector *connector) -{ - if (!drm_mode_create_aspect_ratio_property(connector->dev)) - drm_object_attach_property(&connector->base, - connector->dev->mode_config.aspect_ratio_property, - DRM_MODE_PICTURE_ASPECT_NONE); -} - -static void intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *connector) { intel_attach_force_audio_property(connector); @@ -1702,6 +2032,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum port port = intel_dig_port->port; + uint8_t alternate_ddc_pin; drm_connector_init(dev, connector, &intel_hdmi_connector_funcs, DRM_MODE_CONNECTOR_HDMIA); @@ -1717,7 +2048,14 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_hdmi->ddc_bus = GMBUS_PIN_1_BXT; else intel_hdmi->ddc_bus = GMBUS_PIN_DPB; - intel_encoder->hpd_pin = HPD_PORT_B; + /* + * On BXT A0/A1, sw needs to activate DDIA HPD logic and + * interrupts to check the external panel connection. + */ + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) + intel_encoder->hpd_pin = HPD_PORT_A; + else + intel_encoder->hpd_pin = HPD_PORT_B; break; case PORT_C: if (IS_BROXTON(dev_priv)) @@ -1735,6 +2073,26 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_hdmi->ddc_bus = GMBUS_PIN_DPD; intel_encoder->hpd_pin = HPD_PORT_D; break; + case PORT_E: + /* On SKL PORT E doesn't have seperate GMBUS pin + * We rely on VBT to set a proper alternate GMBUS pin. */ + alternate_ddc_pin = + dev_priv->vbt.ddi_port_info[PORT_E].alternate_ddc_pin; + switch (alternate_ddc_pin) { + case DDC_PIN_B: + intel_hdmi->ddc_bus = GMBUS_PIN_DPB; + break; + case DDC_PIN_C: + intel_hdmi->ddc_bus = GMBUS_PIN_DPC; + break; + case DDC_PIN_D: + intel_hdmi->ddc_bus = GMBUS_PIN_DPD; + break; + default: + MISSING_CASE(alternate_ddc_pin); + } + intel_encoder->hpd_pin = HPD_PORT_E; + break; case PORT_A: intel_encoder->hpd_pin = HPD_PORT_A; /* Internal port only for eDP. */ @@ -1774,6 +2132,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_connector_attach_encoder(intel_connector, intel_encoder); drm_connector_register(connector); + intel_hdmi->attached_connector = intel_connector; /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written * 0xd. Failure to do so will result in spurious interrupts being @@ -1785,7 +2144,8 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, } } -void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port) +void intel_hdmi_init(struct drm_device *dev, + i915_reg_t hdmi_reg, enum port port) { struct intel_digital_port *intel_dig_port; struct intel_encoder *intel_encoder; @@ -1804,7 +2164,7 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port) intel_encoder = &intel_dig_port->base; drm_encoder_init(dev, &intel_encoder->base, &intel_hdmi_enc_funcs, - DRM_MODE_ENCODER_TMDS); + DRM_MODE_ENCODER_TMDS, NULL); intel_encoder->compute_config = intel_hdmi_compute_config; if (HAS_PCH_SPLIT(dev)) { @@ -1820,6 +2180,7 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port) intel_encoder->pre_enable = chv_hdmi_pre_enable; intel_encoder->enable = vlv_enable_hdmi; intel_encoder->post_disable = chv_hdmi_post_disable; + intel_encoder->post_pll_disable = chv_hdmi_post_pll_disable; } else if (IS_VALLEYVIEW(dev)) { intel_encoder->pre_pll_enable = vlv_hdmi_pre_pll_enable; intel_encoder->pre_enable = vlv_hdmi_pre_enable; @@ -1827,7 +2188,12 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port) intel_encoder->post_disable = vlv_hdmi_post_disable; } else { intel_encoder->pre_enable = intel_hdmi_pre_enable; - intel_encoder->enable = intel_enable_hdmi; + if (HAS_PCH_CPT(dev)) + intel_encoder->enable = cpt_enable_hdmi; + else if (HAS_PCH_IBX(dev)) + intel_encoder->enable = ibx_enable_hdmi; + else + intel_encoder->enable = g4x_enable_hdmi; } intel_encoder->type = INTEL_OUTPUT_HDMI; @@ -1850,7 +2216,7 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port) intel_dig_port->port = port; intel_dig_port->hdmi.hdmi_reg = hdmi_reg; - intel_dig_port->dp.output_reg = 0; + intel_dig_port->dp.output_reg = INVALID_MMIO_REG; intel_hdmi_init_connector(intel_dig_port, intel_connector); } diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c new file mode 100644 index 000000000000..b17785719598 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_hotplug.c @@ -0,0 +1,508 @@ +/* + * Copyright © 2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include <linux/kernel.h> + +#include <drm/drmP.h> +#include <drm/i915_drm.h> + +#include "i915_drv.h" +#include "intel_drv.h" + +/** + * DOC: Hotplug + * + * Simply put, hotplug occurs when a display is connected to or disconnected + * from the system. However, there may be adapters and docking stations and + * Display Port short pulses and MST devices involved, complicating matters. + * + * Hotplug in i915 is handled in many different levels of abstraction. + * + * The platform dependent interrupt handling code in i915_irq.c enables, + * disables, and does preliminary handling of the interrupts. The interrupt + * handlers gather the hotplug detect (HPD) information from relevant registers + * into a platform independent mask of hotplug pins that have fired. + * + * The platform independent interrupt handler intel_hpd_irq_handler() in + * intel_hotplug.c does hotplug irq storm detection and mitigation, and passes + * further processing to appropriate bottom halves (Display Port specific and + * regular hotplug). + * + * The Display Port work function i915_digport_work_func() calls into + * intel_dp_hpd_pulse() via hooks, which handles DP short pulses and DP MST long + * pulses, with failures and non-MST long pulses triggering regular hotplug + * processing on the connector. + * + * The regular hotplug work function i915_hotplug_work_func() calls connector + * detect hooks, and, if connector status changes, triggers sending of hotplug + * uevent to userspace via drm_kms_helper_hotplug_event(). + * + * Finally, the userspace is responsible for triggering a modeset upon receiving + * the hotplug uevent, disabling or enabling the crtc as needed. + * + * The hotplug interrupt storm detection and mitigation code keeps track of the + * number of interrupts per hotplug pin per a period of time, and if the number + * of interrupts exceeds a certain threshold, the interrupt is disabled for a + * while before being re-enabled. The intention is to mitigate issues raising + * from broken hardware triggering massive amounts of interrupts and grinding + * the system to a halt. + * + * Current implementation expects that hotplug interrupt storm will not be + * seen when display port sink is connected, hence on platforms whose DP + * callback is handled by i915_digport_work_func reenabling of hpd is not + * performed (it was never expected to be disabled in the first place ;) ) + * this is specific to DP sinks handled by this routine and any other display + * such as HDMI or DVI enabled on the same port will have proper logic since + * it will use i915_hotplug_work_func where this logic is handled. + */ + +bool intel_hpd_pin_to_port(enum hpd_pin pin, enum port *port) +{ + switch (pin) { + case HPD_PORT_A: + *port = PORT_A; + return true; + case HPD_PORT_B: + *port = PORT_B; + return true; + case HPD_PORT_C: + *port = PORT_C; + return true; + case HPD_PORT_D: + *port = PORT_D; + return true; + case HPD_PORT_E: + *port = PORT_E; + return true; + default: + return false; /* no hpd */ + } +} + +#define HPD_STORM_DETECT_PERIOD 1000 +#define HPD_STORM_THRESHOLD 5 +#define HPD_STORM_REENABLE_DELAY (2 * 60 * 1000) + +/** + * intel_hpd_irq_storm_detect - gather stats and detect HPD irq storm on a pin + * @dev_priv: private driver data pointer + * @pin: the pin to gather stats on + * + * Gather stats about HPD irqs from the specified @pin, and detect irq + * storms. Only the pin specific stats and state are changed, the caller is + * responsible for further action. + * + * @HPD_STORM_THRESHOLD irqs are allowed within @HPD_STORM_DETECT_PERIOD ms, + * otherwise it's considered an irq storm, and the irq state is set to + * @HPD_MARK_DISABLED. + * + * Return true if an irq storm was detected on @pin. + */ +static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv, + enum hpd_pin pin) +{ + unsigned long start = dev_priv->hotplug.stats[pin].last_jiffies; + unsigned long end = start + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD); + bool storm = false; + + if (!time_in_range(jiffies, start, end)) { + dev_priv->hotplug.stats[pin].last_jiffies = jiffies; + dev_priv->hotplug.stats[pin].count = 0; + DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", pin); + } else if (dev_priv->hotplug.stats[pin].count > HPD_STORM_THRESHOLD) { + dev_priv->hotplug.stats[pin].state = HPD_MARK_DISABLED; + DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", pin); + storm = true; + } else { + dev_priv->hotplug.stats[pin].count++; + DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", pin, + dev_priv->hotplug.stats[pin].count); + } + + return storm; +} + +static void intel_hpd_irq_storm_disable(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct drm_mode_config *mode_config = &dev->mode_config; + struct intel_connector *intel_connector; + struct intel_encoder *intel_encoder; + struct drm_connector *connector; + enum hpd_pin pin; + bool hpd_disabled = false; + + assert_spin_locked(&dev_priv->irq_lock); + + list_for_each_entry(connector, &mode_config->connector_list, head) { + if (connector->polled != DRM_CONNECTOR_POLL_HPD) + continue; + + intel_connector = to_intel_connector(connector); + intel_encoder = intel_connector->encoder; + if (!intel_encoder) + continue; + + pin = intel_encoder->hpd_pin; + if (pin == HPD_NONE || + dev_priv->hotplug.stats[pin].state != HPD_MARK_DISABLED) + continue; + + DRM_INFO("HPD interrupt storm detected on connector %s: " + "switching from hotplug detection to polling\n", + connector->name); + + dev_priv->hotplug.stats[pin].state = HPD_DISABLED; + connector->polled = DRM_CONNECTOR_POLL_CONNECT + | DRM_CONNECTOR_POLL_DISCONNECT; + hpd_disabled = true; + } + + /* Enable polling and queue hotplug re-enabling. */ + if (hpd_disabled) { + drm_kms_helper_poll_enable_locked(dev); + mod_delayed_work(system_wq, &dev_priv->hotplug.reenable_work, + msecs_to_jiffies(HPD_STORM_REENABLE_DELAY)); + } +} + +static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, typeof(*dev_priv), + hotplug.reenable_work.work); + struct drm_device *dev = dev_priv->dev; + struct drm_mode_config *mode_config = &dev->mode_config; + int i; + + intel_runtime_pm_get(dev_priv); + + spin_lock_irq(&dev_priv->irq_lock); + for_each_hpd_pin(i) { + struct drm_connector *connector; + + if (dev_priv->hotplug.stats[i].state != HPD_DISABLED) + continue; + + dev_priv->hotplug.stats[i].state = HPD_ENABLED; + + list_for_each_entry(connector, &mode_config->connector_list, head) { + struct intel_connector *intel_connector = to_intel_connector(connector); + + if (intel_connector->encoder->hpd_pin == i) { + if (connector->polled != intel_connector->polled) + DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n", + connector->name); + connector->polled = intel_connector->polled; + if (!connector->polled) + connector->polled = DRM_CONNECTOR_POLL_HPD; + } + } + } + if (dev_priv->display.hpd_irq_setup) + dev_priv->display.hpd_irq_setup(dev); + spin_unlock_irq(&dev_priv->irq_lock); + + intel_runtime_pm_put(dev_priv); +} + +static bool intel_hpd_irq_event(struct drm_device *dev, + struct drm_connector *connector) +{ + enum drm_connector_status old_status; + + WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); + old_status = connector->status; + + connector->status = connector->funcs->detect(connector, false); + if (old_status == connector->status) + return false; + + DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n", + connector->base.id, + connector->name, + drm_get_connector_status_name(old_status), + drm_get_connector_status_name(connector->status)); + + return true; +} + +static void i915_digport_work_func(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, hotplug.dig_port_work); + u32 long_port_mask, short_port_mask; + struct intel_digital_port *intel_dig_port; + int i; + u32 old_bits = 0; + + spin_lock_irq(&dev_priv->irq_lock); + long_port_mask = dev_priv->hotplug.long_port_mask; + dev_priv->hotplug.long_port_mask = 0; + short_port_mask = dev_priv->hotplug.short_port_mask; + dev_priv->hotplug.short_port_mask = 0; + spin_unlock_irq(&dev_priv->irq_lock); + + for (i = 0; i < I915_MAX_PORTS; i++) { + bool valid = false; + bool long_hpd = false; + intel_dig_port = dev_priv->hotplug.irq_port[i]; + if (!intel_dig_port || !intel_dig_port->hpd_pulse) + continue; + + if (long_port_mask & (1 << i)) { + valid = true; + long_hpd = true; + } else if (short_port_mask & (1 << i)) + valid = true; + + if (valid) { + enum irqreturn ret; + + ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd); + if (ret == IRQ_NONE) { + /* fall back to old school hpd */ + old_bits |= (1 << intel_dig_port->base.hpd_pin); + } + } + } + + if (old_bits) { + spin_lock_irq(&dev_priv->irq_lock); + dev_priv->hotplug.event_bits |= old_bits; + spin_unlock_irq(&dev_priv->irq_lock); + schedule_work(&dev_priv->hotplug.hotplug_work); + } +} + +/* + * Handle hotplug events outside the interrupt handler proper. + */ +static void i915_hotplug_work_func(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, hotplug.hotplug_work); + struct drm_device *dev = dev_priv->dev; + struct drm_mode_config *mode_config = &dev->mode_config; + struct intel_connector *intel_connector; + struct intel_encoder *intel_encoder; + struct drm_connector *connector; + bool changed = false; + u32 hpd_event_bits; + + mutex_lock(&mode_config->mutex); + DRM_DEBUG_KMS("running encoder hotplug functions\n"); + + spin_lock_irq(&dev_priv->irq_lock); + + hpd_event_bits = dev_priv->hotplug.event_bits; + dev_priv->hotplug.event_bits = 0; + + /* Disable hotplug on connectors that hit an irq storm. */ + intel_hpd_irq_storm_disable(dev_priv); + + spin_unlock_irq(&dev_priv->irq_lock); + + list_for_each_entry(connector, &mode_config->connector_list, head) { + intel_connector = to_intel_connector(connector); + if (!intel_connector->encoder) + continue; + intel_encoder = intel_connector->encoder; + if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) { + DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n", + connector->name, intel_encoder->hpd_pin); + if (intel_encoder->hot_plug) + intel_encoder->hot_plug(intel_encoder); + if (intel_hpd_irq_event(dev, connector)) + changed = true; + } + } + mutex_unlock(&mode_config->mutex); + + if (changed) + drm_kms_helper_hotplug_event(dev); +} + + +/** + * intel_hpd_irq_handler - main hotplug irq handler + * @dev: drm device + * @pin_mask: a mask of hpd pins that have triggered the irq + * @long_mask: a mask of hpd pins that may be long hpd pulses + * + * This is the main hotplug irq handler for all platforms. The platform specific + * irq handlers call the platform specific hotplug irq handlers, which read and + * decode the appropriate registers into bitmasks about hpd pins that have + * triggered (@pin_mask), and which of those pins may be long pulses + * (@long_mask). The @long_mask is ignored if the port corresponding to the pin + * is not a digital port. + * + * Here, we do hotplug irq storm detection and mitigation, and pass further + * processing to appropriate bottom halves. + */ +void intel_hpd_irq_handler(struct drm_device *dev, + u32 pin_mask, u32 long_mask) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int i; + enum port port; + bool storm_detected = false; + bool queue_dig = false, queue_hp = false; + bool is_dig_port; + + if (!pin_mask) + return; + + spin_lock(&dev_priv->irq_lock); + for_each_hpd_pin(i) { + if (!(BIT(i) & pin_mask)) + continue; + + is_dig_port = intel_hpd_pin_to_port(i, &port) && + dev_priv->hotplug.irq_port[port]; + + if (is_dig_port) { + bool long_hpd = long_mask & BIT(i); + + DRM_DEBUG_DRIVER("digital hpd port %c - %s\n", port_name(port), + long_hpd ? "long" : "short"); + /* + * For long HPD pulses we want to have the digital queue happen, + * but we still want HPD storm detection to function. + */ + queue_dig = true; + if (long_hpd) { + dev_priv->hotplug.long_port_mask |= (1 << port); + } else { + /* for short HPD just trigger the digital queue */ + dev_priv->hotplug.short_port_mask |= (1 << port); + continue; + } + } + + if (dev_priv->hotplug.stats[i].state == HPD_DISABLED) { + /* + * On GMCH platforms the interrupt mask bits only + * prevent irq generation, not the setting of the + * hotplug bits itself. So only WARN about unexpected + * interrupts on saner platforms. + */ + WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev), + "Received HPD interrupt on pin %d although disabled\n", i); + continue; + } + + if (dev_priv->hotplug.stats[i].state != HPD_ENABLED) + continue; + + if (!is_dig_port) { + dev_priv->hotplug.event_bits |= BIT(i); + queue_hp = true; + } + + if (intel_hpd_irq_storm_detect(dev_priv, i)) { + dev_priv->hotplug.event_bits &= ~BIT(i); + storm_detected = true; + } + } + + if (storm_detected) + dev_priv->display.hpd_irq_setup(dev); + spin_unlock(&dev_priv->irq_lock); + + /* + * Our hotplug handler can grab modeset locks (by calling down into the + * fb helpers). Hence it must not be run on our own dev-priv->wq work + * queue for otherwise the flush_work in the pageflip code will + * deadlock. + */ + if (queue_dig) + queue_work(dev_priv->hotplug.dp_wq, &dev_priv->hotplug.dig_port_work); + if (queue_hp) + schedule_work(&dev_priv->hotplug.hotplug_work); +} + +/** + * intel_hpd_init - initializes and enables hpd support + * @dev_priv: i915 device instance + * + * This function enables the hotplug support. It requires that interrupts have + * already been enabled with intel_irq_init_hw(). From this point on hotplug and + * poll request can run concurrently to other code, so locking rules must be + * obeyed. + * + * This is a separate step from interrupt enabling to simplify the locking rules + * in the driver load and resume code. + */ +void intel_hpd_init(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct drm_mode_config *mode_config = &dev->mode_config; + struct drm_connector *connector; + int i; + + for_each_hpd_pin(i) { + dev_priv->hotplug.stats[i].count = 0; + dev_priv->hotplug.stats[i].state = HPD_ENABLED; + } + list_for_each_entry(connector, &mode_config->connector_list, head) { + struct intel_connector *intel_connector = to_intel_connector(connector); + connector->polled = intel_connector->polled; + if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE) + connector->polled = DRM_CONNECTOR_POLL_HPD; + if (intel_connector->mst_port) + connector->polled = DRM_CONNECTOR_POLL_HPD; + } + + /* + * Interrupt setup is already guaranteed to be single-threaded, this is + * just to make the assert_spin_locked checks happy. + */ + spin_lock_irq(&dev_priv->irq_lock); + if (dev_priv->display.hpd_irq_setup) + dev_priv->display.hpd_irq_setup(dev); + spin_unlock_irq(&dev_priv->irq_lock); +} + +void intel_hpd_init_work(struct drm_i915_private *dev_priv) +{ + INIT_WORK(&dev_priv->hotplug.hotplug_work, i915_hotplug_work_func); + INIT_WORK(&dev_priv->hotplug.dig_port_work, i915_digport_work_func); + INIT_DELAYED_WORK(&dev_priv->hotplug.reenable_work, + intel_hpd_irq_storm_reenable_work); +} + +void intel_hpd_cancel_work(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + + dev_priv->hotplug.long_port_mask = 0; + dev_priv->hotplug.short_port_mask = 0; + dev_priv->hotplug.event_bits = 0; + + spin_unlock_irq(&dev_priv->irq_lock); + + cancel_work_sync(&dev_priv->hotplug.dig_port_work); + cancel_work_sync(&dev_priv->hotplug.hotplug_work); + cancel_delayed_work_sync(&dev_priv->hotplug.reenable_work); +} diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c index a64f26c670af..e26e22a72e3b 100644 --- a/drivers/gpu/drm/i915/intel_i2c.c +++ b/drivers/gpu/drm/i915/intel_i2c.c @@ -36,7 +36,7 @@ struct gmbus_pin { const char *name; - int reg; + i915_reg_t reg; }; /* Map gmbus pin pairs to names and registers. */ @@ -63,9 +63,9 @@ static const struct gmbus_pin gmbus_pins_skl[] = { }; static const struct gmbus_pin gmbus_pins_bxt[] = { - [GMBUS_PIN_1_BXT] = { "dpb", PCH_GPIOB }, - [GMBUS_PIN_2_BXT] = { "dpc", PCH_GPIOC }, - [GMBUS_PIN_3_BXT] = { "misc", PCH_GPIOD }, + [GMBUS_PIN_1_BXT] = { "dpb", GPIOB }, + [GMBUS_PIN_2_BXT] = { "dpc", GPIOC }, + [GMBUS_PIN_3_BXT] = { "misc", GPIOD }, }; /* pin is expected to be valid */ @@ -74,7 +74,7 @@ static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *dev_priv, { if (IS_BROXTON(dev_priv)) return &gmbus_pins_bxt[pin]; - else if (IS_SKYLAKE(dev_priv)) + else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) return &gmbus_pins_skl[pin]; else if (IS_BROADWELL(dev_priv)) return &gmbus_pins_bdw[pin]; @@ -89,14 +89,15 @@ bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv, if (IS_BROXTON(dev_priv)) size = ARRAY_SIZE(gmbus_pins_bxt); - else if (IS_SKYLAKE(dev_priv)) + else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) size = ARRAY_SIZE(gmbus_pins_skl); else if (IS_BROADWELL(dev_priv)) size = ARRAY_SIZE(gmbus_pins_bdw); else size = ARRAY_SIZE(gmbus_pins); - return pin < size && get_gmbus_pin(dev_priv, pin)->reg; + return pin < size && + i915_mmio_reg_valid(get_gmbus_pin(dev_priv, pin)->reg); } /* Intel GPIO access functions */ @@ -114,8 +115,8 @@ intel_i2c_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(dev_priv->gpio_mmio_base + GMBUS0, 0); - I915_WRITE(dev_priv->gpio_mmio_base + GMBUS4, 0); + I915_WRITE(GMBUS0, 0); + I915_WRITE(GMBUS4, 0); } static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable) @@ -240,9 +241,8 @@ intel_gpio_setup(struct intel_gmbus *bus, unsigned int pin) algo = &bus->bit_algo; - bus->gpio_reg = dev_priv->gpio_mmio_base + - get_gmbus_pin(dev_priv, pin)->reg; - + bus->gpio_reg = _MMIO(dev_priv->gpio_mmio_base + + i915_mmio_reg_offset(get_gmbus_pin(dev_priv, pin)->reg)); bus->adapter.algo_data = algo; algo->setsda = set_data; algo->setscl = set_clock; @@ -261,7 +261,6 @@ gmbus_wait_hw_status(struct drm_i915_private *dev_priv, u32 gmbus4_irq_en) { int i; - int reg_offset = dev_priv->gpio_mmio_base; u32 gmbus2 = 0; DEFINE_WAIT(wait); @@ -271,13 +270,13 @@ gmbus_wait_hw_status(struct drm_i915_private *dev_priv, /* Important: The hw handles only the first bit, so set only one! Since * we also need to check for NAKs besides the hw ready/idle signal, we * need to wake up periodically and check that ourselves. */ - I915_WRITE(GMBUS4 + reg_offset, gmbus4_irq_en); + I915_WRITE(GMBUS4, gmbus4_irq_en); for (i = 0; i < msecs_to_jiffies_timeout(50); i++) { prepare_to_wait(&dev_priv->gmbus_wait_queue, &wait, TASK_UNINTERRUPTIBLE); - gmbus2 = I915_READ_NOTRACE(GMBUS2 + reg_offset); + gmbus2 = I915_READ_NOTRACE(GMBUS2); if (gmbus2 & (GMBUS_SATOER | gmbus2_status)) break; @@ -285,7 +284,7 @@ gmbus_wait_hw_status(struct drm_i915_private *dev_priv, } finish_wait(&dev_priv->gmbus_wait_queue, &wait); - I915_WRITE(GMBUS4 + reg_offset, 0); + I915_WRITE(GMBUS4, 0); if (gmbus2 & GMBUS_SATOER) return -ENXIO; @@ -298,20 +297,19 @@ static int gmbus_wait_idle(struct drm_i915_private *dev_priv) { int ret; - int reg_offset = dev_priv->gpio_mmio_base; -#define C ((I915_READ_NOTRACE(GMBUS2 + reg_offset) & GMBUS_ACTIVE) == 0) +#define C ((I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0) if (!HAS_GMBUS_IRQ(dev_priv->dev)) return wait_for(C, 10); /* Important: The hw handles only the first bit, so set only one! */ - I915_WRITE(GMBUS4 + reg_offset, GMBUS_IDLE_EN); + I915_WRITE(GMBUS4, GMBUS_IDLE_EN); ret = wait_event_timeout(dev_priv->gmbus_wait_queue, C, msecs_to_jiffies_timeout(10)); - I915_WRITE(GMBUS4 + reg_offset, 0); + I915_WRITE(GMBUS4, 0); if (ret) return 0; @@ -325,9 +323,7 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv, unsigned short addr, u8 *buf, unsigned int len, u32 gmbus1_index) { - int reg_offset = dev_priv->gpio_mmio_base; - - I915_WRITE(GMBUS1 + reg_offset, + I915_WRITE(GMBUS1, gmbus1_index | GMBUS_CYCLE_WAIT | (len << GMBUS_BYTE_COUNT_SHIFT) | @@ -342,7 +338,7 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv, if (ret) return ret; - val = I915_READ(GMBUS3 + reg_offset); + val = I915_READ(GMBUS3); do { *buf++ = val & 0xff; val >>= 8; @@ -380,7 +376,6 @@ static int gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, unsigned short addr, u8 *buf, unsigned int len) { - int reg_offset = dev_priv->gpio_mmio_base; unsigned int chunk_size = len; u32 val, loop; @@ -390,8 +385,8 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, len -= 1; } - I915_WRITE(GMBUS3 + reg_offset, val); - I915_WRITE(GMBUS1 + reg_offset, + I915_WRITE(GMBUS3, val); + I915_WRITE(GMBUS1, GMBUS_CYCLE_WAIT | (chunk_size << GMBUS_BYTE_COUNT_SHIFT) | (addr << GMBUS_SLAVE_ADDR_SHIFT) | @@ -404,7 +399,7 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, val |= *buf++ << (8 * loop); } while (--len && ++loop < 4); - I915_WRITE(GMBUS3 + reg_offset, val); + I915_WRITE(GMBUS3, val); ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN); @@ -452,7 +447,6 @@ gmbus_is_index_read(struct i2c_msg *msgs, int i, int num) static int gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct i2c_msg *msgs) { - int reg_offset = dev_priv->gpio_mmio_base; u32 gmbus1_index = 0; u32 gmbus5 = 0; int ret; @@ -466,41 +460,29 @@ gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct i2c_msg *msgs) /* GMBUS5 holds 16-bit index */ if (gmbus5) - I915_WRITE(GMBUS5 + reg_offset, gmbus5); + I915_WRITE(GMBUS5, gmbus5); ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus1_index); /* Clear GMBUS5 after each index transfer */ if (gmbus5) - I915_WRITE(GMBUS5 + reg_offset, 0); + I915_WRITE(GMBUS5, 0); return ret; } static int -gmbus_xfer(struct i2c_adapter *adapter, - struct i2c_msg *msgs, - int num) +do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) { struct intel_gmbus *bus = container_of(adapter, struct intel_gmbus, adapter); struct drm_i915_private *dev_priv = bus->dev_priv; - int i = 0, inc, try = 0, reg_offset; + int i = 0, inc, try = 0; int ret = 0; - intel_aux_display_runtime_get(dev_priv); - mutex_lock(&dev_priv->gmbus_mutex); - - if (bus->force_bit) { - ret = i2c_bit_algo.master_xfer(adapter, msgs, num); - goto out; - } - - reg_offset = dev_priv->gpio_mmio_base; - retry: - I915_WRITE(GMBUS0 + reg_offset, bus->reg0); + I915_WRITE(GMBUS0, bus->reg0); for (; i < num; i += inc) { inc = 1; @@ -513,24 +495,20 @@ retry: ret = gmbus_xfer_write(dev_priv, &msgs[i]); } + if (!ret) + ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_WAIT_PHASE, + GMBUS_HW_WAIT_EN); if (ret == -ETIMEDOUT) goto timeout; - if (ret == -ENXIO) + else if (ret) goto clear_err; - - ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_WAIT_PHASE, - GMBUS_HW_WAIT_EN); - if (ret == -ENXIO) - goto clear_err; - if (ret) - goto timeout; } /* Generate a STOP condition on the bus. Note that gmbus can't generata * a STOP on the very first cycle. To simplify the code we * unconditionally generate the STOP condition with an additional gmbus * cycle. */ - I915_WRITE(GMBUS1 + reg_offset, GMBUS_CYCLE_STOP | GMBUS_SW_RDY); + I915_WRITE(GMBUS1, GMBUS_CYCLE_STOP | GMBUS_SW_RDY); /* Mark the GMBUS interface as disabled after waiting for idle. * We will re-enable it at the start of the next xfer, @@ -541,7 +519,7 @@ retry: adapter->name); ret = -ETIMEDOUT; } - I915_WRITE(GMBUS0 + reg_offset, 0); + I915_WRITE(GMBUS0, 0); ret = ret ?: i; goto out; @@ -570,9 +548,9 @@ clear_err: * of resetting the GMBUS controller and so clearing the * BUS_ERROR raised by the slave's NAK. */ - I915_WRITE(GMBUS1 + reg_offset, GMBUS_SW_CLR_INT); - I915_WRITE(GMBUS1 + reg_offset, 0); - I915_WRITE(GMBUS0 + reg_offset, 0); + I915_WRITE(GMBUS1, GMBUS_SW_CLR_INT); + I915_WRITE(GMBUS1, 0); + I915_WRITE(GMBUS0, 0); DRM_DEBUG_KMS("GMBUS [%s] NAK for addr: %04x %c(%d)\n", adapter->name, msgs[i].addr, @@ -595,15 +573,38 @@ clear_err: timeout: DRM_INFO("GMBUS [%s] timed out, falling back to bit banging on pin %d\n", bus->adapter.name, bus->reg0 & 0xff); - I915_WRITE(GMBUS0 + reg_offset, 0); + I915_WRITE(GMBUS0, 0); - /* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */ + /* + * Hardware may not support GMBUS over these pins? Try GPIO bitbanging + * instead. Use EAGAIN to have i2c core retry. + */ bus->force_bit = 1; - ret = i2c_bit_algo.master_xfer(adapter, msgs, num); + ret = -EAGAIN; out: + return ret; +} + +static int +gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) +{ + struct intel_gmbus *bus = container_of(adapter, struct intel_gmbus, + adapter); + struct drm_i915_private *dev_priv = bus->dev_priv; + int ret; + + intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + mutex_lock(&dev_priv->gmbus_mutex); + + if (bus->force_bit) + ret = i2c_bit_algo.master_xfer(adapter, msgs, num); + else + ret = do_gmbus_xfer(adapter, msgs, num); + mutex_unlock(&dev_priv->gmbus_mutex); - intel_aux_display_runtime_put(dev_priv); + intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); + return ret; } @@ -634,12 +635,13 @@ int intel_setup_gmbus(struct drm_device *dev) if (HAS_PCH_NOP(dev)) return 0; - else if (HAS_PCH_SPLIT(dev)) - dev_priv->gpio_mmio_base = PCH_GPIOA - GPIOA; - else if (IS_VALLEYVIEW(dev)) + + if (IS_VALLEYVIEW(dev)) dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE; - else - dev_priv->gpio_mmio_base = 0; + else if (!HAS_GMCH_DISPLAY(dev_priv)) + dev_priv->gpio_mmio_base = + i915_mmio_reg_offset(PCH_GPIOA) - + i915_mmio_reg_offset(GPIOA); mutex_init(&dev_priv->gmbus_mutex); init_waitqueue_head(&dev_priv->gmbus_wait_queue); diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 7f2161a1ff5d..4ebafab53f30 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -135,6 +135,7 @@ #include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" +#include "intel_mocs.h" #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE) @@ -189,21 +190,32 @@ #define GEN8_CTX_L3LLC_COHERENT (1<<5) #define GEN8_CTX_PRIVILEGE (1<<8) -#define ASSIGN_CTX_PDP(ppgtt, reg_state, n) { \ - const u64 _addr = test_bit(n, ppgtt->pdp.used_pdpes) ? \ - ppgtt->pdp.page_directory[n]->daddr : \ - ppgtt->scratch_pd->daddr; \ +#define ASSIGN_CTX_REG(reg_state, pos, reg, val) do { \ + (reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \ + (reg_state)[(pos)+1] = (val); \ +} while (0) + +#define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do { \ + const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n)); \ reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \ reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \ -} +} while (0) + +#define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \ + reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \ + reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \ +} while (0) enum { ADVANCED_CONTEXT = 0, - LEGACY_CONTEXT, + LEGACY_32B_CONTEXT, ADVANCED_AD_CONTEXT, LEGACY_64B_CONTEXT }; -#define GEN8_CTX_MODE_SHIFT 3 +#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 +#define GEN8_CTX_ADDRESSING_MODE(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ + LEGACY_64B_CONTEXT :\ + LEGACY_32B_CONTEXT) enum { FAULT_AND_HANG = 0, FAULT_AND_HALT, /* Debug only */ @@ -211,9 +223,12 @@ enum { FAULT_AND_CONTINUE /* Unsupported */ }; #define GEN8_CTX_ID_SHIFT 32 +#define CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 + +static int intel_lr_context_pin(struct drm_i915_gem_request *rq); +static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring, + struct drm_i915_gem_object *default_ctx_obj); -static int intel_lr_context_pin(struct intel_engine_cs *ring, - struct intel_context *ctx); /** * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists @@ -229,6 +244,12 @@ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists { WARN_ON(i915.enable_ppgtt == -1); + /* On platforms with execlist available, vGPU will only + * support execlist mode, no ring buffer mode. + */ + if (HAS_LOGICAL_RING_CONTEXTS(dev) && intel_vgpu_active(dev)) + return 1; + if (INTEL_INFO(dev)->gen >= 9) return 1; @@ -256,24 +277,35 @@ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists */ u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj) { - u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj); + u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj) + + LRC_PPHWSP_PN * PAGE_SIZE; /* LRCA is required to be 4K aligned so the more significant 20 bits * are globally unique */ return lrca >> 12; } -static uint64_t execlists_ctx_descriptor(struct intel_engine_cs *ring, - struct drm_i915_gem_object *ctx_obj) +static bool disable_lite_restore_wa(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; + + return (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) && + (ring->id == VCS || ring->id == VCS2); +} + +uint64_t intel_lr_context_descriptor(struct intel_context *ctx, + struct intel_engine_cs *ring) +{ + struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state; uint64_t desc; - uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj); + uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj) + + LRC_PPHWSP_PN * PAGE_SIZE; WARN_ON(lrca & 0xFFFFFFFF00000FFFULL); desc = GEN8_CTX_VALID; - desc |= LEGACY_CONTEXT << GEN8_CTX_MODE_SHIFT; + desc |= GEN8_CTX_ADDRESSING_MODE(dev) << GEN8_CTX_ADDRESSING_MODE_SHIFT; if (IS_GEN8(ctx_obj->base.dev)) desc |= GEN8_CTX_L3LLC_COHERENT; desc |= GEN8_CTX_PRIVILEGE; @@ -285,69 +317,73 @@ static uint64_t execlists_ctx_descriptor(struct intel_engine_cs *ring, /* desc |= GEN8_CTX_FORCE_RESTORE; */ /* WaEnableForceRestoreInCtxtDescForVCS:skl */ - if (IS_GEN9(dev) && - INTEL_REVID(dev) <= SKL_REVID_B0 && - (ring->id == BCS || ring->id == VCS || - ring->id == VECS || ring->id == VCS2)) + /* WaEnableForceRestoreInCtxtDescForVCS:bxt */ + if (disable_lite_restore_wa(ring)) desc |= GEN8_CTX_FORCE_RESTORE; return desc; } -static void execlists_elsp_write(struct intel_engine_cs *ring, - struct drm_i915_gem_object *ctx_obj0, - struct drm_i915_gem_object *ctx_obj1) +static void execlists_elsp_write(struct drm_i915_gem_request *rq0, + struct drm_i915_gem_request *rq1) { + + struct intel_engine_cs *ring = rq0->ring; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - uint64_t temp = 0; - uint32_t desc[4]; + uint64_t desc[2]; - /* XXX: You must always write both descriptors in the order below. */ - if (ctx_obj1) - temp = execlists_ctx_descriptor(ring, ctx_obj1); - else - temp = 0; - desc[1] = (u32)(temp >> 32); - desc[0] = (u32)temp; + if (rq1) { + desc[1] = intel_lr_context_descriptor(rq1->ctx, rq1->ring); + rq1->elsp_submitted++; + } else { + desc[1] = 0; + } - temp = execlists_ctx_descriptor(ring, ctx_obj0); - desc[3] = (u32)(temp >> 32); - desc[2] = (u32)temp; + desc[0] = intel_lr_context_descriptor(rq0->ctx, rq0->ring); + rq0->elsp_submitted++; + /* You must always write both descriptors in the order below. */ spin_lock(&dev_priv->uncore.lock); intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL); - I915_WRITE_FW(RING_ELSP(ring), desc[1]); - I915_WRITE_FW(RING_ELSP(ring), desc[0]); - I915_WRITE_FW(RING_ELSP(ring), desc[3]); + I915_WRITE_FW(RING_ELSP(ring), upper_32_bits(desc[1])); + I915_WRITE_FW(RING_ELSP(ring), lower_32_bits(desc[1])); + I915_WRITE_FW(RING_ELSP(ring), upper_32_bits(desc[0])); /* The context is automatically loaded after the following */ - I915_WRITE_FW(RING_ELSP(ring), desc[2]); + I915_WRITE_FW(RING_ELSP(ring), lower_32_bits(desc[0])); - /* ELSP is a wo register, so use another nearby reg for posting instead */ - POSTING_READ_FW(RING_EXECLIST_STATUS(ring)); + /* ELSP is a wo register, use another nearby reg for posting */ + POSTING_READ_FW(RING_EXECLIST_STATUS_LO(ring)); intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL); spin_unlock(&dev_priv->uncore.lock); } -static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, - struct drm_i915_gem_object *ring_obj, - struct i915_hw_ppgtt *ppgtt, - u32 tail) +static int execlists_update_context(struct drm_i915_gem_request *rq) { + struct intel_engine_cs *ring = rq->ring; + struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt; + struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state; + struct drm_i915_gem_object *rb_obj = rq->ringbuf->obj; struct page *page; uint32_t *reg_state; - page = i915_gem_object_get_page(ctx_obj, 1); + BUG_ON(!ctx_obj); + WARN_ON(!i915_gem_obj_is_pinned(ctx_obj)); + WARN_ON(!i915_gem_obj_is_pinned(rb_obj)); + + page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); reg_state = kmap_atomic(page); - reg_state[CTX_RING_TAIL+1] = tail; - reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj); + reg_state[CTX_RING_TAIL+1] = rq->tail; + reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj); - /* True PPGTT with dynamic page allocation: update PDP registers and - * point the unallocated PDPs to the scratch page - */ - if (ppgtt) { + if (ppgtt && !USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { + /* True 32b PPGTT with dynamic page allocation: update PDP + * registers and point the unallocated PDPs to scratch page. + * PML4 is allocated during ppgtt init, so this is not needed + * in 48-bit mode. + */ ASSIGN_CTX_PDP(ppgtt, reg_state, 3); ASSIGN_CTX_PDP(ppgtt, reg_state, 2); ASSIGN_CTX_PDP(ppgtt, reg_state, 1); @@ -359,32 +395,15 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, return 0; } -static void execlists_submit_contexts(struct intel_engine_cs *ring, - struct intel_context *to0, u32 tail0, - struct intel_context *to1, u32 tail1) +static void execlists_submit_requests(struct drm_i915_gem_request *rq0, + struct drm_i915_gem_request *rq1) { - struct drm_i915_gem_object *ctx_obj0 = to0->engine[ring->id].state; - struct intel_ringbuffer *ringbuf0 = to0->engine[ring->id].ringbuf; - struct drm_i915_gem_object *ctx_obj1 = NULL; - struct intel_ringbuffer *ringbuf1 = NULL; - - BUG_ON(!ctx_obj0); - WARN_ON(!i915_gem_obj_is_pinned(ctx_obj0)); - WARN_ON(!i915_gem_obj_is_pinned(ringbuf0->obj)); - - execlists_update_context(ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0); + execlists_update_context(rq0); - if (to1) { - ringbuf1 = to1->engine[ring->id].ringbuf; - ctx_obj1 = to1->engine[ring->id].state; - BUG_ON(!ctx_obj1); - WARN_ON(!i915_gem_obj_is_pinned(ctx_obj1)); - WARN_ON(!i915_gem_obj_is_pinned(ringbuf1->obj)); + if (rq1) + execlists_update_context(rq1); - execlists_update_context(ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); - } - - execlists_elsp_write(ring, ctx_obj0, ctx_obj1); + execlists_elsp_write(rq0, rq1); } static void execlists_context_unqueue(struct intel_engine_cs *ring) @@ -444,13 +463,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *ring) WARN_ON(req1 && req1->elsp_submitted); - execlists_submit_contexts(ring, req0->ctx, req0->tail, - req1 ? req1->ctx : NULL, - req1 ? req1->tail : 0); - - req0->elsp_submitted++; - if (req1) - req1->elsp_submitted++; + execlists_submit_requests(req0, req1); } static bool execlists_check_remove_request(struct intel_engine_cs *ring, @@ -496,25 +509,26 @@ void intel_lrc_irq_handler(struct intel_engine_cs *ring) u32 status_pointer; u8 read_pointer; u8 write_pointer; - u32 status; + u32 status = 0; u32 status_id; u32 submit_contexts = 0; status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring)); read_pointer = ring->next_context_status_buffer; - write_pointer = status_pointer & 0x07; + write_pointer = status_pointer & GEN8_CSB_PTR_MASK; if (read_pointer > write_pointer) - write_pointer += 6; + write_pointer += GEN8_CSB_ENTRIES; spin_lock(&ring->execlist_lock); while (read_pointer < write_pointer) { read_pointer++; - status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + - (read_pointer % 6) * 8); - status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + - (read_pointer % 6) * 8 + 4); + status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, read_pointer % GEN8_CSB_ENTRIES)); + status_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, read_pointer % GEN8_CSB_ENTRIES)); + + if (status & GEN8_CTX_STATUS_IDLE_ACTIVE) + continue; if (status & GEN8_CTX_STATUS_PREEMPTED) { if (status & GEN8_CTX_STATUS_LITE_RESTORE) { @@ -531,46 +545,36 @@ void intel_lrc_irq_handler(struct intel_engine_cs *ring) } } - if (submit_contexts != 0) + if (disable_lite_restore_wa(ring)) { + /* Prevent a ctx to preempt itself */ + if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) && + (submit_contexts != 0)) + execlists_context_unqueue(ring); + } else if (submit_contexts != 0) { execlists_context_unqueue(ring); + } spin_unlock(&ring->execlist_lock); WARN(submit_contexts > 2, "More than two context complete events?\n"); - ring->next_context_status_buffer = write_pointer % 6; + ring->next_context_status_buffer = write_pointer % GEN8_CSB_ENTRIES; I915_WRITE(RING_CONTEXT_STATUS_PTR(ring), - ((u32)ring->next_context_status_buffer & 0x07) << 8); + _MASKED_FIELD(GEN8_CSB_PTR_MASK << 8, + ((u32)ring->next_context_status_buffer & + GEN8_CSB_PTR_MASK) << 8)); } -static int execlists_context_queue(struct intel_engine_cs *ring, - struct intel_context *to, - u32 tail, - struct drm_i915_gem_request *request) +static int execlists_context_queue(struct drm_i915_gem_request *request) { + struct intel_engine_cs *ring = request->ring; struct drm_i915_gem_request *cursor; int num_elements = 0; - if (to != ring->default_context) - intel_lr_context_pin(ring, to); + if (request->ctx != ring->default_context) + intel_lr_context_pin(request); - if (!request) { - /* - * If there isn't a request associated with this submission, - * create one as a temporary holder. - */ - request = kzalloc(sizeof(*request), GFP_KERNEL); - if (request == NULL) - return -ENOMEM; - request->ring = ring; - request->ctx = to; - kref_init(&request->ref); - i915_gem_context_reference(request->ctx); - } else { - i915_gem_request_reference(request); - WARN_ON(to != request->ctx); - } - request->tail = tail; + i915_gem_request_reference(request); spin_lock_irq(&ring->execlist_lock); @@ -585,7 +589,7 @@ static int execlists_context_queue(struct intel_engine_cs *ring, struct drm_i915_gem_request, execlist_link); - if (to == tail_req->ctx) { + if (request->ctx == tail_req->ctx) { WARN(tail_req->elsp_submitted != 0, "More than 2 already-submitted reqs queued\n"); list_del(&tail_req->execlist_link); @@ -603,10 +607,9 @@ static int execlists_context_queue(struct intel_engine_cs *ring, return 0; } -static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx) +static int logical_ring_invalidate_all_caches(struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring = ringbuf->ring; + struct intel_engine_cs *ring = req->ring; uint32_t flush_domains; int ret; @@ -614,8 +617,7 @@ static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf, if (ring->gpu_caches_dirty) flush_domains = I915_GEM_GPU_DOMAINS; - ret = ring->emit_flush(ringbuf, ctx, - I915_GEM_GPU_DOMAINS, flush_domains); + ret = ring->emit_flush(req, I915_GEM_GPU_DOMAINS, flush_domains); if (ret) return ret; @@ -623,12 +625,10 @@ static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf, return 0; } -static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, +static int execlists_move_to_gpu(struct drm_i915_gem_request *req, struct list_head *vmas) { - struct intel_engine_cs *ring = ringbuf->ring; - const unsigned other_rings = ~intel_ring_flag(ring); + const unsigned other_rings = ~intel_ring_flag(req->ring); struct i915_vma *vma; uint32_t flush_domains = 0; bool flush_chipset = false; @@ -638,7 +638,7 @@ static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf, struct drm_i915_gem_object *obj = vma->obj; if (obj->active & other_rings) { - ret = i915_gem_object_sync(obj, ring); + ret = i915_gem_object_sync(obj, req->ring, &req); if (ret) return ret; } @@ -655,59 +655,59 @@ static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf, /* Unconditionally invalidate gpu caches and ensure that we do flush * any residual writes from the previous batch. */ - return logical_ring_invalidate_all_caches(ringbuf, ctx); + return logical_ring_invalidate_all_caches(req); } -int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request, - struct intel_context *ctx) +int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) { int ret; - if (ctx != request->ring->default_context) { - ret = intel_lr_context_pin(request->ring, ctx); + request->ringbuf = request->ctx->engine[request->ring->id].ringbuf; + + if (request->ctx != request->ring->default_context) { + ret = intel_lr_context_pin(request); if (ret) return ret; } - request->ringbuf = ctx->engine[request->ring->id].ringbuf; - request->ctx = ctx; - i915_gem_context_reference(request->ctx); - return 0; } -static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, +static int logical_ring_wait_for_space(struct drm_i915_gem_request *req, int bytes) { - struct intel_engine_cs *ring = ringbuf->ring; - struct drm_i915_gem_request *request; + struct intel_ringbuffer *ringbuf = req->ringbuf; + struct intel_engine_cs *ring = req->ring; + struct drm_i915_gem_request *target; unsigned space; int ret; if (intel_ring_space(ringbuf) >= bytes) return 0; - list_for_each_entry(request, &ring->request_list, list) { + /* The whole point of reserving space is to not wait! */ + WARN_ON(ringbuf->reserved_in_use); + + list_for_each_entry(target, &ring->request_list, list) { /* * The request queue is per-engine, so can contain requests * from multiple ringbuffers. Here, we must ignore any that * aren't from the ringbuffer we're considering. */ - if (request->ringbuf != ringbuf) + if (target->ringbuf != ringbuf) continue; /* Would completion of this request free enough space? */ - space = __intel_ring_space(request->postfix, ringbuf->tail, + space = __intel_ring_space(target->postfix, ringbuf->tail, ringbuf->size); if (space >= bytes) break; } - if (WARN_ON(&request->list == &ring->request_list)) + if (WARN_ON(&target->list == &ring->request_list)) return -ENOSPC; - ret = i915_wait_request(request); + ret = i915_wait_request(target); if (ret) return ret; @@ -717,7 +717,7 @@ static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf, /* * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload - * @ringbuf: Logical Ringbuffer to advance. + * @request: Request to advance the logical ringbuffer of. * * The tail is updated in our logical ringbuffer struct, not in the actual context. What * really happens during submission is that the context and current tail will be placed @@ -725,33 +725,29 @@ static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf, * point, the tail *inside* the context is updated and the ELSP written to. */ static void -intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, - struct drm_i915_gem_request *request) +intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) { - struct intel_engine_cs *ring = ringbuf->ring; + struct intel_engine_cs *ring = request->ring; + struct drm_i915_private *dev_priv = request->i915; - intel_logical_ring_advance(ringbuf); + intel_logical_ring_advance(request->ringbuf); + + request->tail = request->ringbuf->tail; if (intel_ring_stopped(ring)) return; - execlists_context_queue(ring, ctx, ringbuf->tail, request); + if (dev_priv->guc.execbuf_client) + i915_guc_submit(dev_priv->guc.execbuf_client, request); + else + execlists_context_queue(request); } -static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx) +static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf) { uint32_t __iomem *virt; int rem = ringbuf->size - ringbuf->tail; - if (ringbuf->space < rem) { - int ret = logical_ring_wait_for_space(ringbuf, ctx, rem); - - if (ret) - return ret; - } - virt = ringbuf->virtual_start + ringbuf->tail; rem /= 4; while (rem--) @@ -759,25 +755,50 @@ static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf, ringbuf->tail = 0; intel_ring_update_space(ringbuf); - - return 0; } -static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, int bytes) +static int logical_ring_prepare(struct drm_i915_gem_request *req, int bytes) { - int ret; + struct intel_ringbuffer *ringbuf = req->ringbuf; + int remain_usable = ringbuf->effective_size - ringbuf->tail; + int remain_actual = ringbuf->size - ringbuf->tail; + int ret, total_bytes, wait_bytes = 0; + bool need_wrap = false; + + if (ringbuf->reserved_in_use) + total_bytes = bytes; + else + total_bytes = bytes + ringbuf->reserved_size; - if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) { - ret = logical_ring_wrap_buffer(ringbuf, ctx); - if (unlikely(ret)) - return ret; + if (unlikely(bytes > remain_usable)) { + /* + * Not enough space for the basic request. So need to flush + * out the remainder and then wait for base + reserved. + */ + wait_bytes = remain_actual + total_bytes; + need_wrap = true; + } else { + if (unlikely(total_bytes > remain_usable)) { + /* + * The base request will fit but the reserved space + * falls off the end. So only need to to wait for the + * reserved size after flushing out the remainder. + */ + wait_bytes = remain_actual + ringbuf->reserved_size; + need_wrap = true; + } else if (total_bytes > ringbuf->space) { + /* No wrapping required, just waiting. */ + wait_bytes = total_bytes; + } } - if (unlikely(ringbuf->space < bytes)) { - ret = logical_ring_wait_for_space(ringbuf, ctx, bytes); + if (wait_bytes) { + ret = logical_ring_wait_for_space(req, wait_bytes); if (unlikely(ret)) return ret; + + if (need_wrap) + __wrap_ring_buffer(ringbuf); } return 0; @@ -786,7 +807,7 @@ static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, /** * intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands * - * @ringbuf: Logical ringbuffer. + * @req: The request to start some new work for * @num_dwords: number of DWORDs that we plan to write to the ringbuffer. * * The ringbuffer might not be ready to accept the commands right away (maybe it needs to @@ -796,32 +817,42 @@ static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, * * Return: non-zero if the ringbuffer is not ready to be written to. */ -static int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, int num_dwords) +int intel_logical_ring_begin(struct drm_i915_gem_request *req, int num_dwords) { - struct intel_engine_cs *ring = ringbuf->ring; - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv; int ret; + WARN_ON(req == NULL); + dev_priv = req->ring->dev->dev_private; + ret = i915_gem_check_wedge(&dev_priv->gpu_error, dev_priv->mm.interruptible); if (ret) return ret; - ret = logical_ring_prepare(ringbuf, ctx, num_dwords * sizeof(uint32_t)); - if (ret) - return ret; - - /* Preallocate the olr before touching the ring */ - ret = i915_gem_request_alloc(ring, ctx); + ret = logical_ring_prepare(req, num_dwords * sizeof(uint32_t)); if (ret) return ret; - ringbuf->space -= num_dwords * sizeof(uint32_t); + req->ringbuf->space -= num_dwords * sizeof(uint32_t); return 0; } +int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request) +{ + /* + * The first call merely notes the reserve request and is common for + * all back ends. The subsequent localised _begin() call actually + * ensures that the reservation is available. Without the begin, if + * the request creator immediately submitted the request without + * adding any commands to it then there might not actually be + * sufficient room for the submission commands. + */ + intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); + + return intel_logical_ring_begin(request, 0); +} + /** * execlists_submission() - submit a batchbuffer for execution, Execlists style * @dev: DRM device. @@ -839,16 +870,15 @@ static int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, * * Return: non-zero if the submission fails. */ -int intel_execlists_submission(struct drm_device *dev, struct drm_file *file, - struct intel_engine_cs *ring, - struct intel_context *ctx, +int intel_execlists_submission(struct i915_execbuffer_params *params, struct drm_i915_gem_execbuffer2 *args, - struct list_head *vmas, - struct drm_i915_gem_object *batch_obj, - u64 exec_start, u32 dispatch_flags) + struct list_head *vmas) { + struct drm_device *dev = params->dev; + struct intel_engine_cs *ring = params->ring; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; + struct intel_ringbuffer *ringbuf = params->ctx->engine[ring->id].ringbuf; + u64 exec_start; int instp_mode; u32 instp_mask; int ret; @@ -879,53 +909,41 @@ int intel_execlists_submission(struct drm_device *dev, struct drm_file *file, return -EINVAL; } - if (args->num_cliprects != 0) { - DRM_DEBUG("clip rectangles are only valid on pre-gen5\n"); - return -EINVAL; - } else { - if (args->DR4 == 0xffffffff) { - DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); - args->DR4 = 0; - } - - if (args->DR1 || args->DR4 || args->cliprects_ptr) { - DRM_DEBUG("0 cliprects but dirt in cliprects fields\n"); - return -EINVAL; - } - } - if (args->flags & I915_EXEC_GEN7_SOL_RESET) { DRM_DEBUG("sol reset is gen7 only\n"); return -EINVAL; } - ret = execlists_move_to_gpu(ringbuf, ctx, vmas); + ret = execlists_move_to_gpu(params->request, vmas); if (ret) return ret; if (ring == &dev_priv->ring[RCS] && instp_mode != dev_priv->relative_constants_mode) { - ret = intel_logical_ring_begin(ringbuf, ctx, 4); + ret = intel_logical_ring_begin(params->request, 4); if (ret) return ret; intel_logical_ring_emit(ringbuf, MI_NOOP); intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1)); - intel_logical_ring_emit(ringbuf, INSTPM); + intel_logical_ring_emit_reg(ringbuf, INSTPM); intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode); intel_logical_ring_advance(ringbuf); dev_priv->relative_constants_mode = instp_mode; } - ret = ring->emit_bb_start(ringbuf, ctx, exec_start, dispatch_flags); + exec_start = params->batch_obj_vm_offset + + args->batch_start_offset; + + ret = ring->emit_bb_start(params->request, exec_start, params->dispatch_flags); if (ret) return ret; - trace_i915_gem_ring_dispatch(intel_ring_get_request(ring), dispatch_flags); + trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags); - i915_gem_execbuffer_move_to_active(vmas, ring); - i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); + i915_gem_execbuffer_move_to_active(vmas, params->request); + i915_gem_execbuffer_retire_commands(params); return 0; } @@ -950,7 +968,7 @@ void intel_execlists_retire_requests(struct intel_engine_cs *ring) ctx->engine[ring->id].state; if (ctx_obj && (ctx != ring->default_context)) - intel_lr_context_unpin(ring, ctx); + intel_lr_context_unpin(req); list_del(&req->execlist_link); i915_gem_request_unreference(req); } @@ -978,16 +996,15 @@ void intel_logical_ring_stop(struct intel_engine_cs *ring) I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); } -int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx) +int logical_ring_flush_all_caches(struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring = ringbuf->ring; + struct intel_engine_cs *ring = req->ring; int ret; if (!ring->gpu_caches_dirty) return 0; - ret = ring->emit_flush(ringbuf, ctx, 0, I915_GEM_GPU_DOMAINS); + ret = ring->emit_flush(req, 0, I915_GEM_GPU_DOMAINS); if (ret) return ret; @@ -995,57 +1012,77 @@ int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf, return 0; } -static int intel_lr_context_pin(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int intel_lr_context_do_pin(struct intel_engine_cs *ring, + struct drm_i915_gem_object *ctx_obj, + struct intel_ringbuffer *ringbuf) { - struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state; - struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; int ret = 0; WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex)); - if (ctx->engine[ring->id].pin_count++ == 0) { - ret = i915_gem_obj_ggtt_pin(ctx_obj, - GEN8_LR_CONTEXT_ALIGN, 0); - if (ret) - goto reset_pin_count; + ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, + PIN_OFFSET_BIAS | GUC_WOPCM_TOP); + if (ret) + return ret; - ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf); - if (ret) - goto unpin_ctx_obj; + ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf); + if (ret) + goto unpin_ctx_obj; - ctx_obj->dirty = true; - } + ctx_obj->dirty = true; + + /* Invalidate GuC TLB. */ + if (i915.enable_guc_submission) + I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); return ret; unpin_ctx_obj: i915_gem_object_ggtt_unpin(ctx_obj); -reset_pin_count: - ctx->engine[ring->id].pin_count = 0; return ret; } -void intel_lr_context_unpin(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int intel_lr_context_pin(struct drm_i915_gem_request *rq) { - struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state; - struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; + int ret = 0; + struct intel_engine_cs *ring = rq->ring; + struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state; + struct intel_ringbuffer *ringbuf = rq->ringbuf; + + if (rq->ctx->engine[ring->id].pin_count++ == 0) { + ret = intel_lr_context_do_pin(ring, ctx_obj, ringbuf); + if (ret) + goto reset_pin_count; + } + return ret; + +reset_pin_count: + rq->ctx->engine[ring->id].pin_count = 0; + return ret; +} + +void intel_lr_context_unpin(struct drm_i915_gem_request *rq) +{ + struct intel_engine_cs *ring = rq->ring; + struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state; + struct intel_ringbuffer *ringbuf = rq->ringbuf; if (ctx_obj) { WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex)); - if (--ctx->engine[ring->id].pin_count == 0) { + if (--rq->ctx->engine[ring->id].pin_count == 0) { intel_unpin_ringbuffer_obj(ringbuf); i915_gem_object_ggtt_unpin(ctx_obj); } } } -static int intel_logical_ring_workarounds_emit(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) { int ret, i; - struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; + struct intel_engine_cs *ring = req->ring; + struct intel_ringbuffer *ringbuf = req->ringbuf; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct i915_workarounds *w = &dev_priv->workarounds; @@ -1054,17 +1091,17 @@ static int intel_logical_ring_workarounds_emit(struct intel_engine_cs *ring, return 0; ring->gpu_caches_dirty = true; - ret = logical_ring_flush_all_caches(ringbuf, ctx); + ret = logical_ring_flush_all_caches(req); if (ret) return ret; - ret = intel_logical_ring_begin(ringbuf, ctx, w->count * 2 + 2); + ret = intel_logical_ring_begin(req, w->count * 2 + 2); if (ret) return ret; intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count)); for (i = 0; i < w->count; i++) { - intel_logical_ring_emit(ringbuf, w->reg[i].addr); + intel_logical_ring_emit_reg(ringbuf, w->reg[i].addr); intel_logical_ring_emit(ringbuf, w->reg[i].value); } intel_logical_ring_emit(ringbuf, MI_NOOP); @@ -1072,32 +1109,403 @@ static int intel_logical_ring_workarounds_emit(struct intel_engine_cs *ring, intel_logical_ring_advance(ringbuf); ring->gpu_caches_dirty = true; - ret = logical_ring_flush_all_caches(ringbuf, ctx); + ret = logical_ring_flush_all_caches(req); if (ret) return ret; return 0; } +#define wa_ctx_emit(batch, index, cmd) \ + do { \ + int __index = (index)++; \ + if (WARN_ON(__index >= (PAGE_SIZE / sizeof(uint32_t)))) { \ + return -ENOSPC; \ + } \ + batch[__index] = (cmd); \ + } while (0) + +#define wa_ctx_emit_reg(batch, index, reg) \ + wa_ctx_emit((batch), (index), i915_mmio_reg_offset(reg)) + +/* + * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after + * PIPE_CONTROL instruction. This is required for the flush to happen correctly + * but there is a slight complication as this is applied in WA batch where the + * values are only initialized once so we cannot take register value at the + * beginning and reuse it further; hence we save its value to memory, upload a + * constant value with bit21 set and then we restore it back with the saved value. + * To simplify the WA, a constant value is formed by using the default value + * of this register. This shouldn't be a problem because we are only modifying + * it for a short period and this batch in non-premptible. We can ofcourse + * use additional instructions that read the actual value of the register + * at that time and set our bit of interest but it makes the WA complicated. + * + * This WA is also required for Gen9 so extracting as a function avoids + * code duplication. + */ +static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *ring, + uint32_t *const batch, + uint32_t index) +{ + uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES); + + /* + * WaDisableLSQCROPERFforOCL:skl + * This WA is implemented in skl_init_clock_gating() but since + * this batch updates GEN8_L3SQCREG4 with default value we need to + * set this bit here to retain the WA during flush. + */ + if (IS_SKL_REVID(ring->dev, 0, SKL_REVID_E0)) + l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS; + + wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT)); + wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4); + wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256); + wa_ctx_emit(batch, index, 0); + + wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1)); + wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4); + wa_ctx_emit(batch, index, l3sqc4_flush); + + wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6)); + wa_ctx_emit(batch, index, (PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_DC_FLUSH_ENABLE)); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + + wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT)); + wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4); + wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256); + wa_ctx_emit(batch, index, 0); + + return index; +} + +static inline uint32_t wa_ctx_start(struct i915_wa_ctx_bb *wa_ctx, + uint32_t offset, + uint32_t start_alignment) +{ + return wa_ctx->offset = ALIGN(offset, start_alignment); +} + +static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx, + uint32_t offset, + uint32_t size_alignment) +{ + wa_ctx->size = offset - wa_ctx->offset; + + WARN(wa_ctx->size % size_alignment, + "wa_ctx_bb failed sanity checks: size %d is not aligned to %d\n", + wa_ctx->size, size_alignment); + return 0; +} + +/** + * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA + * + * @ring: only applicable for RCS + * @wa_ctx: structure representing wa_ctx + * offset: specifies start of the batch, should be cache-aligned. This is updated + * with the offset value received as input. + * size: size of the batch in DWORDS but HW expects in terms of cachelines + * @batch: page in which WA are loaded + * @offset: This field specifies the start of the batch, it should be + * cache-aligned otherwise it is adjusted accordingly. + * Typically we only have one indirect_ctx and per_ctx batch buffer which are + * initialized at the beginning and shared across all contexts but this field + * helps us to have multiple batches at different offsets and select them based + * on a criteria. At the moment this batch always start at the beginning of the page + * and at this point we don't have multiple wa_ctx batch buffers. + * + * The number of WA applied are not known at the beginning; we use this field + * to return the no of DWORDS written. + * + * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END + * so it adds NOOPs as padding to make it cacheline aligned. + * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together + * makes a complete batch buffer. + * + * Return: non-zero if we exceed the PAGE_SIZE limit. + */ + +static int gen8_init_indirectctx_bb(struct intel_engine_cs *ring, + struct i915_wa_ctx_bb *wa_ctx, + uint32_t *const batch, + uint32_t *offset) +{ + uint32_t scratch_addr; + uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); + + /* WaDisableCtxRestoreArbitration:bdw,chv */ + wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); + + /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ + if (IS_BROADWELL(ring->dev)) { + int rc = gen8_emit_flush_coherentl3_wa(ring, batch, index); + if (rc < 0) + return rc; + index = rc; + } + + /* WaClearSlmSpaceAtContextSwitch:bdw,chv */ + /* Actual scratch location is at 128 bytes offset */ + scratch_addr = ring->scratch.gtt_offset + 2*CACHELINE_BYTES; + + wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6)); + wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 | + PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_QW_WRITE)); + wa_ctx_emit(batch, index, scratch_addr); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + + /* Pad to end of cacheline */ + while (index % CACHELINE_DWORDS) + wa_ctx_emit(batch, index, MI_NOOP); + + /* + * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because + * execution depends on the length specified in terms of cache lines + * in the register CTX_RCS_INDIRECT_CTX + */ + + return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS); +} + +/** + * gen8_init_perctx_bb() - initialize per ctx batch with WA + * + * @ring: only applicable for RCS + * @wa_ctx: structure representing wa_ctx + * offset: specifies start of the batch, should be cache-aligned. + * size: size of the batch in DWORDS but HW expects in terms of cachelines + * @batch: page in which WA are loaded + * @offset: This field specifies the start of this batch. + * This batch is started immediately after indirect_ctx batch. Since we ensure + * that indirect_ctx ends on a cacheline this batch is aligned automatically. + * + * The number of DWORDS written are returned using this field. + * + * This batch is terminated with MI_BATCH_BUFFER_END and so we need not add padding + * to align it with cacheline as padding after MI_BATCH_BUFFER_END is redundant. + */ +static int gen8_init_perctx_bb(struct intel_engine_cs *ring, + struct i915_wa_ctx_bb *wa_ctx, + uint32_t *const batch, + uint32_t *offset) +{ + uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); + + /* WaDisableCtxRestoreArbitration:bdw,chv */ + wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE); + + wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END); + + return wa_ctx_end(wa_ctx, *offset = index, 1); +} + +static int gen9_init_indirectctx_bb(struct intel_engine_cs *ring, + struct i915_wa_ctx_bb *wa_ctx, + uint32_t *const batch, + uint32_t *offset) +{ + int ret; + struct drm_device *dev = ring->dev; + uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); + + /* WaDisableCtxRestoreArbitration:skl,bxt */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); + + /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */ + ret = gen8_emit_flush_coherentl3_wa(ring, batch, index); + if (ret < 0) + return ret; + index = ret; + + /* Pad to end of cacheline */ + while (index % CACHELINE_DWORDS) + wa_ctx_emit(batch, index, MI_NOOP); + + return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS); +} + +static int gen9_init_perctx_bb(struct intel_engine_cs *ring, + struct i915_wa_ctx_bb *wa_ctx, + uint32_t *const batch, + uint32_t *offset) +{ + struct drm_device *dev = ring->dev; + uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); + + /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1)); + wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0); + wa_ctx_emit(batch, index, + _MASKED_BIT_ENABLE(DISABLE_PIXEL_MASK_CAMMING)); + wa_ctx_emit(batch, index, MI_NOOP); + } + + /* WaDisableCtxRestoreArbitration:skl,bxt */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE); + + wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END); + + return wa_ctx_end(wa_ctx, *offset = index, 1); +} + +static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *ring, u32 size) +{ + int ret; + + ring->wa_ctx.obj = i915_gem_alloc_object(ring->dev, PAGE_ALIGN(size)); + if (!ring->wa_ctx.obj) { + DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n"); + return -ENOMEM; + } + + ret = i915_gem_obj_ggtt_pin(ring->wa_ctx.obj, PAGE_SIZE, 0); + if (ret) { + DRM_DEBUG_DRIVER("pin LRC WA ctx backing obj failed: %d\n", + ret); + drm_gem_object_unreference(&ring->wa_ctx.obj->base); + return ret; + } + + return 0; +} + +static void lrc_destroy_wa_ctx_obj(struct intel_engine_cs *ring) +{ + if (ring->wa_ctx.obj) { + i915_gem_object_ggtt_unpin(ring->wa_ctx.obj); + drm_gem_object_unreference(&ring->wa_ctx.obj->base); + ring->wa_ctx.obj = NULL; + } +} + +static int intel_init_workaround_bb(struct intel_engine_cs *ring) +{ + int ret; + uint32_t *batch; + uint32_t offset; + struct page *page; + struct i915_ctx_workarounds *wa_ctx = &ring->wa_ctx; + + WARN_ON(ring->id != RCS); + + /* update this when WA for higher Gen are added */ + if (INTEL_INFO(ring->dev)->gen > 9) { + DRM_ERROR("WA batch buffer is not initialized for Gen%d\n", + INTEL_INFO(ring->dev)->gen); + return 0; + } + + /* some WA perform writes to scratch page, ensure it is valid */ + if (ring->scratch.obj == NULL) { + DRM_ERROR("scratch page not allocated for %s\n", ring->name); + return -EINVAL; + } + + ret = lrc_setup_wa_ctx_obj(ring, PAGE_SIZE); + if (ret) { + DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret); + return ret; + } + + page = i915_gem_object_get_page(wa_ctx->obj, 0); + batch = kmap_atomic(page); + offset = 0; + + if (INTEL_INFO(ring->dev)->gen == 8) { + ret = gen8_init_indirectctx_bb(ring, + &wa_ctx->indirect_ctx, + batch, + &offset); + if (ret) + goto out; + + ret = gen8_init_perctx_bb(ring, + &wa_ctx->per_ctx, + batch, + &offset); + if (ret) + goto out; + } else if (INTEL_INFO(ring->dev)->gen == 9) { + ret = gen9_init_indirectctx_bb(ring, + &wa_ctx->indirect_ctx, + batch, + &offset); + if (ret) + goto out; + + ret = gen9_init_perctx_bb(ring, + &wa_ctx->per_ctx, + batch, + &offset); + if (ret) + goto out; + } + +out: + kunmap_atomic(batch); + if (ret) + lrc_destroy_wa_ctx_obj(ring); + + return ret; +} + static int gen8_init_common_ring(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; + u8 next_context_status_buffer_hw; + + lrc_setup_hardware_status_page(ring, + ring->default_context->engine[ring->id].state); I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask)); I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff); - if (ring->status_page.obj) { - I915_WRITE(RING_HWS_PGA(ring->mmio_base), - (u32)ring->status_page.gfx_addr); - POSTING_READ(RING_HWS_PGA(ring->mmio_base)); - } - I915_WRITE(RING_MODE_GEN7(ring), _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) | _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE)); POSTING_READ(RING_MODE_GEN7(ring)); - ring->next_context_status_buffer = 0; + + /* + * Instead of resetting the Context Status Buffer (CSB) read pointer to + * zero, we need to read the write pointer from hardware and use its + * value because "this register is power context save restored". + * Effectively, these states have been observed: + * + * | Suspend-to-idle (freeze) | Suspend-to-RAM (mem) | + * BDW | CSB regs not reset | CSB regs reset | + * CHT | CSB regs not reset | CSB regs not reset | + */ + next_context_status_buffer_hw = (I915_READ(RING_CONTEXT_STATUS_PTR(ring)) + & GEN8_CSB_PTR_MASK); + + /* + * When the CSB registers are reset (also after power-up / gpu reset), + * CSB write pointer is set to all 1's, which is not valid, use '5' in + * this special case, so the first element read is CSB[0]. + */ + if (next_context_status_buffer_hw == GEN8_CSB_PTR_MASK) + next_context_status_buffer_hw = (GEN8_CSB_ENTRIES - 1); + + ring->next_context_status_buffer = next_context_status_buffer_hw; DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name); memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); @@ -1139,19 +1547,68 @@ static int gen9_init_render_ring(struct intel_engine_cs *ring) return init_workarounds_ring(ring); } -static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, +static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req) +{ + struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt; + struct intel_engine_cs *ring = req->ring; + struct intel_ringbuffer *ringbuf = req->ringbuf; + const int num_lri_cmds = GEN8_LEGACY_PDPES * 2; + int i, ret; + + ret = intel_logical_ring_begin(req, num_lri_cmds * 2 + 2); + if (ret) + return ret; + + intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(num_lri_cmds)); + for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { + const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); + + intel_logical_ring_emit_reg(ringbuf, GEN8_RING_PDP_UDW(ring, i)); + intel_logical_ring_emit(ringbuf, upper_32_bits(pd_daddr)); + intel_logical_ring_emit_reg(ringbuf, GEN8_RING_PDP_LDW(ring, i)); + intel_logical_ring_emit(ringbuf, lower_32_bits(pd_daddr)); + } + + intel_logical_ring_emit(ringbuf, MI_NOOP); + intel_logical_ring_advance(ringbuf); + + return 0; +} + +static int gen8_emit_bb_start(struct drm_i915_gem_request *req, u64 offset, unsigned dispatch_flags) { + struct intel_ringbuffer *ringbuf = req->ringbuf; bool ppgtt = !(dispatch_flags & I915_DISPATCH_SECURE); int ret; - ret = intel_logical_ring_begin(ringbuf, ctx, 4); + /* Don't rely in hw updating PDPs, specially in lite-restore. + * Ideally, we should set Force PD Restore in ctx descriptor, + * but we can't. Force Restore would be a second option, but + * it is unsafe in case of lite-restore (because the ctx is + * not idle). PML4 is allocated during ppgtt init so this is + * not needed in 48-bit.*/ + if (req->ctx->ppgtt && + (intel_ring_flag(req->ring) & req->ctx->ppgtt->pd_dirty_rings)) { + if (!USES_FULL_48BIT_PPGTT(req->i915) && + !intel_vgpu_active(req->i915->dev)) { + ret = intel_logical_ring_emit_pdps(req); + if (ret) + return ret; + } + + req->ctx->ppgtt->pd_dirty_rings &= ~intel_ring_flag(req->ring); + } + + ret = intel_logical_ring_begin(req, 4); if (ret) return ret; /* FIXME(BDW): Address space and security selectors. */ - intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); + intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | + (ppgtt<<8) | + (dispatch_flags & I915_DISPATCH_RS ? + MI_BATCH_RESOURCE_STREAMER : 0)); intel_logical_ring_emit(ringbuf, lower_32_bits(offset)); intel_logical_ring_emit(ringbuf, upper_32_bits(offset)); intel_logical_ring_emit(ringbuf, MI_NOOP); @@ -1193,18 +1650,18 @@ static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring) spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } -static int gen8_emit_flush(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, +static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 invalidate_domains, u32 unused) { + struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_engine_cs *ring = ringbuf->ring; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; uint32_t cmd; int ret; - ret = intel_logical_ring_begin(ringbuf, ctx, 4); + ret = intel_logical_ring_begin(request, 4); if (ret) return ret; @@ -1234,11 +1691,11 @@ static int gen8_emit_flush(struct intel_ringbuffer *ringbuf, return 0; } -static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, +static int gen8_emit_flush_render(struct drm_i915_gem_request *request, u32 invalidate_domains, u32 flush_domains) { + struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_engine_cs *ring = ringbuf->ring; u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; bool vf_flush_wa; @@ -1250,6 +1707,7 @@ static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf, if (flush_domains) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; + flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (invalidate_domains) { @@ -1270,7 +1728,7 @@ static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf, vf_flush_wa = INTEL_INFO(ring->dev)->gen >= 9 && flags & PIPE_CONTROL_VF_CACHE_INVALIDATE; - ret = intel_logical_ring_begin(ringbuf, ctx, vf_flush_wa ? 12 : 6); + ret = intel_logical_ring_begin(request, vf_flush_wa ? 12 : 6); if (ret) return ret; @@ -1304,9 +1762,37 @@ static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno) intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); } -static int gen8_emit_request(struct intel_ringbuffer *ringbuf, - struct drm_i915_gem_request *request) +static u32 bxt_a_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) { + + /* + * On BXT A steppings there is a HW coherency issue whereby the + * MI_STORE_DATA_IMM storing the completed request's seqno + * occasionally doesn't invalidate the CPU cache. Work around this by + * clflushing the corresponding cacheline whenever the caller wants + * the coherency to be guaranteed. Note that this cacheline is known + * to be clean at this point, since we only write it in + * bxt_a_set_seqno(), where we also do a clflush after the write. So + * this clflush in practice becomes an invalidate operation. + */ + + if (!lazy_coherency) + intel_flush_status_page(ring, I915_GEM_HWS_INDEX); + + return intel_read_status_page(ring, I915_GEM_HWS_INDEX); +} + +static void bxt_a_set_seqno(struct intel_engine_cs *ring, u32 seqno) +{ + intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); + + /* See bxt_a_get_seqno() explaining the reason for the clflush. */ + intel_flush_status_page(ring, I915_GEM_HWS_INDEX); +} + +static int gen8_emit_request(struct drm_i915_gem_request *request) +{ + struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_engine_cs *ring = ringbuf->ring; u32 cmd; int ret; @@ -1316,7 +1802,7 @@ static int gen8_emit_request(struct intel_ringbuffer *ringbuf, * used as a workaround for not being allowed to do lite * restore with HEAD==TAIL (WaIdleLiteRestore). */ - ret = intel_logical_ring_begin(ringbuf, request->ctx, 8); + ret = intel_logical_ring_begin(request, 8); if (ret) return ret; @@ -1328,11 +1814,10 @@ static int gen8_emit_request(struct intel_ringbuffer *ringbuf, (ring->status_page.gfx_addr + (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT))); intel_logical_ring_emit(ringbuf, 0); - intel_logical_ring_emit(ringbuf, - i915_gem_request_get_seqno(ring->outstanding_lazy_request)); + intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT); intel_logical_ring_emit(ringbuf, MI_NOOP); - intel_logical_ring_advance_and_submit(ringbuf, request->ctx, request); + intel_logical_ring_advance_and_submit(request); /* * Here we add two extra NOOPs as padding to avoid @@ -1345,49 +1830,53 @@ static int gen8_emit_request(struct intel_ringbuffer *ringbuf, return 0; } -static int intel_lr_context_render_state_init(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int intel_lr_context_render_state_init(struct drm_i915_gem_request *req) { - struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; struct render_state so; - struct drm_i915_file_private *file_priv = ctx->file_priv; - struct drm_file *file = file_priv ? file_priv->file : NULL; int ret; - ret = i915_gem_render_state_prepare(ring, &so); + ret = i915_gem_render_state_prepare(req->ring, &so); if (ret) return ret; if (so.rodata == NULL) return 0; - ret = ring->emit_bb_start(ringbuf, - ctx, - so.ggtt_offset, - I915_DISPATCH_SECURE); + ret = req->ring->emit_bb_start(req, so.ggtt_offset, + I915_DISPATCH_SECURE); if (ret) goto out; - i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring); + ret = req->ring->emit_bb_start(req, + (so.ggtt_offset + so.aux_batch_offset), + I915_DISPATCH_SECURE); + if (ret) + goto out; + + i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req); - ret = __i915_add_request(ring, file, so.obj); - /* intel_logical_ring_add_request moves object to inactive if it - * fails */ out: i915_gem_render_state_fini(&so); return ret; } -static int gen8_init_rcs_context(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int gen8_init_rcs_context(struct drm_i915_gem_request *req) { int ret; - ret = intel_logical_ring_workarounds_emit(ring, ctx); + ret = intel_logical_ring_workarounds_emit(req); if (ret) return ret; - return intel_lr_context_render_state_init(ring, ctx); + ret = intel_rcs_context_init_mocs(req); + /* + * Failing to program the MOCS is non-fatal.The system will not + * run at peak performance. So generate an error and carry on. + */ + if (ret) + DRM_ERROR("MOCS failed to program: expect performance issues.\n"); + + return intel_lr_context_render_state_init(req); } /** @@ -1407,7 +1896,6 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *ring) intel_logical_ring_stop(ring); WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); - i915_gem_request_assign(&ring->outstanding_lazy_request, NULL); if (ring->cleanup) ring->cleanup(ring); @@ -1419,6 +1907,8 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *ring) kunmap(sg_page(ring->status_page.obj->pages->sgl)); ring->status_page.obj = NULL; } + + lrc_destroy_wa_ctx_obj(ring); } static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring) @@ -1434,6 +1924,7 @@ static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *rin i915_gem_batch_pool_init(dev, &ring->batch_pool); init_waitqueue_head(&ring->irq_queue); + INIT_LIST_HEAD(&ring->buffers); INIT_LIST_HEAD(&ring->execlist_queue); INIT_LIST_HEAD(&ring->execlist_retired_req_list); spin_lock_init(&ring->execlist_lock); @@ -1442,7 +1933,21 @@ static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *rin if (ret) return ret; - ret = intel_lr_context_deferred_create(ring->default_context, ring); + ret = intel_lr_context_deferred_alloc(ring->default_context, ring); + if (ret) + return ret; + + /* As this is the default context, always pin it */ + ret = intel_lr_context_do_pin( + ring, + ring->default_context->engine[ring->id].state, + ring->default_context->engine[ring->id].ringbuf); + if (ret) { + DRM_ERROR( + "Failed to pin and map ringbuffer %s: %d\n", + ring->name, ret); + return ret; + } return ret; } @@ -1469,8 +1974,13 @@ static int logical_render_ring_init(struct drm_device *dev) ring->init_hw = gen8_init_render_ring; ring->init_context = gen8_init_rcs_context; ring->cleanup = intel_fini_pipe_control; - ring->get_seqno = gen8_get_seqno; - ring->set_seqno = gen8_set_seqno; + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + ring->get_seqno = bxt_a_get_seqno; + ring->set_seqno = bxt_a_set_seqno; + } else { + ring->get_seqno = gen8_get_seqno; + ring->set_seqno = gen8_set_seqno; + } ring->emit_request = gen8_emit_request; ring->emit_flush = gen8_emit_flush_render; ring->irq_get = gen8_logical_ring_get_irq; @@ -1478,11 +1988,28 @@ static int logical_render_ring_init(struct drm_device *dev) ring->emit_bb_start = gen8_emit_bb_start; ring->dev = dev; - ret = logical_ring_init(dev, ring); + + ret = intel_init_pipe_control(ring); if (ret) return ret; - return intel_init_pipe_control(ring); + ret = intel_init_workaround_bb(ring); + if (ret) { + /* + * We continue even if we fail to initialize WA batch + * because we only expect rare glitches but nothing + * critical to prevent us from using GPU + */ + DRM_ERROR("WA batch buffer initialization failed: %d\n", + ret); + } + + ret = logical_ring_init(dev, ring); + if (ret) { + lrc_destroy_wa_ctx_obj(ring); + } + + return ret; } static int logical_bsd_ring_init(struct drm_device *dev) @@ -1499,8 +2026,13 @@ static int logical_bsd_ring_init(struct drm_device *dev) GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; ring->init_hw = gen8_init_common_ring; - ring->get_seqno = gen8_get_seqno; - ring->set_seqno = gen8_set_seqno; + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + ring->get_seqno = bxt_a_get_seqno; + ring->set_seqno = bxt_a_set_seqno; + } else { + ring->get_seqno = gen8_get_seqno; + ring->set_seqno = gen8_set_seqno; + } ring->emit_request = gen8_emit_request; ring->emit_flush = gen8_emit_flush; ring->irq_get = gen8_logical_ring_get_irq; @@ -1549,8 +2081,13 @@ static int logical_blt_ring_init(struct drm_device *dev) GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT; ring->init_hw = gen8_init_common_ring; - ring->get_seqno = gen8_get_seqno; - ring->set_seqno = gen8_set_seqno; + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + ring->get_seqno = bxt_a_get_seqno; + ring->set_seqno = bxt_a_set_seqno; + } else { + ring->get_seqno = gen8_get_seqno; + ring->set_seqno = gen8_set_seqno; + } ring->emit_request = gen8_emit_request; ring->emit_flush = gen8_emit_flush; ring->irq_get = gen8_logical_ring_get_irq; @@ -1574,8 +2111,13 @@ static int logical_vebox_ring_init(struct drm_device *dev) GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT; ring->init_hw = gen8_init_common_ring; - ring->get_seqno = gen8_get_seqno; - ring->set_seqno = gen8_set_seqno; + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + ring->get_seqno = bxt_a_get_seqno; + ring->set_seqno = bxt_a_set_seqno; + } else { + ring->get_seqno = gen8_get_seqno; + ring->set_seqno = gen8_set_seqno; + } ring->emit_request = gen8_emit_request; ring->emit_flush = gen8_emit_flush; ring->irq_get = gen8_logical_ring_get_irq; @@ -1628,14 +2170,8 @@ int intel_logical_rings_init(struct drm_device *dev) goto cleanup_vebox_ring; } - ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); - if (ret) - goto cleanup_bsd2_ring; - return 0; -cleanup_bsd2_ring: - intel_logical_ring_cleanup(&dev_priv->ring[VCS2]); cleanup_vebox_ring: intel_logical_ring_cleanup(&dev_priv->ring[VECS]); cleanup_blt_ring: @@ -1721,7 +2257,7 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o /* The second page of the context object contains some fields which must * be set up prior to the first execution. */ - page = i915_gem_object_get_page(ctx_obj, 1); + page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); reg_state = kmap_atomic(page); /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM @@ -1729,73 +2265,81 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o * only for the first context restore: on a subsequent save, the GPU will * recreate this batchbuffer with new values (including all the missing * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */ - if (ring->id == RCS) - reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14); - else - reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11); - reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED; - reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring); - reg_state[CTX_CONTEXT_CONTROL+1] = - _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | - CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); - reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base); - reg_state[CTX_RING_HEAD+1] = 0; - reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base); - reg_state[CTX_RING_TAIL+1] = 0; - reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base); + reg_state[CTX_LRI_HEADER_0] = + MI_LOAD_REGISTER_IMM(ring->id == RCS ? 14 : 11) | MI_LRI_FORCE_POSTED; + ASSIGN_CTX_REG(reg_state, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(ring), + _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | + CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | + CTX_CTRL_RS_CTX_ENABLE)); + ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_RING_TAIL, RING_TAIL(ring->mmio_base), 0); /* Ring buffer start address is not known until the buffer is pinned. * It is written to the context image in execlists_update_context() */ - reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base); - reg_state[CTX_RING_BUFFER_CONTROL+1] = - ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID; - reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168; - reg_state[CTX_BB_HEAD_U+1] = 0; - reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140; - reg_state[CTX_BB_HEAD_L+1] = 0; - reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110; - reg_state[CTX_BB_STATE+1] = (1<<5); - reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c; - reg_state[CTX_SECOND_BB_HEAD_U+1] = 0; - reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114; - reg_state[CTX_SECOND_BB_HEAD_L+1] = 0; - reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118; - reg_state[CTX_SECOND_BB_STATE+1] = 0; + ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_START, RING_START(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_CONTROL, RING_CTL(ring->mmio_base), + ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID); + ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_U, RING_BBADDR_UDW(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_L, RING_BBADDR(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_BB_STATE, RING_BBSTATE(ring->mmio_base), + RING_BB_PPGTT); + ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_STATE, RING_SBBSTATE(ring->mmio_base), 0); if (ring->id == RCS) { - /* TODO: according to BSpec, the register state context - * for CHV does not have these. OTOH, these registers do - * exist in CHV. I'm waiting for a clarification */ - reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0; - reg_state[CTX_BB_PER_CTX_PTR+1] = 0; - reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4; - reg_state[CTX_RCS_INDIRECT_CTX+1] = 0; - reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8; - reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0; - } - reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9); - reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED; - reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8; - reg_state[CTX_CTX_TIMESTAMP+1] = 0; - reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3); - reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3); - reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2); - reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2); - reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1); - reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1); - reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0); - reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0); - - /* With dynamic page allocation, PDPs may not be allocated at this point, - * Point the unallocated PDPs to the scratch page - */ - ASSIGN_CTX_PDP(ppgtt, reg_state, 3); - ASSIGN_CTX_PDP(ppgtt, reg_state, 2); - ASSIGN_CTX_PDP(ppgtt, reg_state, 1); - ASSIGN_CTX_PDP(ppgtt, reg_state, 0); + ASSIGN_CTX_REG(reg_state, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(ring->mmio_base), 0); + ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX_OFFSET, RING_INDIRECT_CTX_OFFSET(ring->mmio_base), 0); + if (ring->wa_ctx.obj) { + struct i915_ctx_workarounds *wa_ctx = &ring->wa_ctx; + uint32_t ggtt_offset = i915_gem_obj_ggtt_offset(wa_ctx->obj); + + reg_state[CTX_RCS_INDIRECT_CTX+1] = + (ggtt_offset + wa_ctx->indirect_ctx.offset * sizeof(uint32_t)) | + (wa_ctx->indirect_ctx.size / CACHELINE_DWORDS); + + reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = + CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT << 6; + + reg_state[CTX_BB_PER_CTX_PTR+1] = + (ggtt_offset + wa_ctx->per_ctx.offset * sizeof(uint32_t)) | + 0x01; + } + } + reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED; + ASSIGN_CTX_REG(reg_state, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(ring->mmio_base), 0); + /* PDP values well be assigned later if needed */ + ASSIGN_CTX_REG(reg_state, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(ring, 3), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(ring, 3), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(ring, 2), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(ring, 2), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(ring, 1), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(ring, 1), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(ring, 0), 0); + ASSIGN_CTX_REG(reg_state, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(ring, 0), 0); + + if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { + /* 64b PPGTT (48bit canonical) + * PDP0_DESCRIPTOR contains the base address to PML4 and + * other PDP Descriptors are ignored. + */ + ASSIGN_CTX_PML4(ppgtt, reg_state); + } else { + /* 32b PPGTT + * PDP*_DESCRIPTOR contains the base address of space supported. + * With dynamic page allocation, PDPs may not be allocated at + * this point. Point the unallocated PDPs to the scratch page + */ + ASSIGN_CTX_PDP(ppgtt, reg_state, 3); + ASSIGN_CTX_PDP(ppgtt, reg_state, 2); + ASSIGN_CTX_PDP(ppgtt, reg_state, 1); + ASSIGN_CTX_PDP(ppgtt, reg_state, 0); + } + if (ring->id == RCS) { reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); - reg_state[CTX_R_PWR_CLK_STATE] = GEN8_R_PWR_CLK_STATE; - reg_state[CTX_R_PWR_CLK_STATE+1] = make_rpcs(dev); + ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, + make_rpcs(dev)); } kunmap_atomic(reg_state); @@ -1832,8 +2376,7 @@ void intel_lr_context_free(struct intel_context *ctx) i915_gem_object_ggtt_unpin(ctx_obj); } WARN_ON(ctx->engine[ring->id].pin_count); - intel_destroy_ringbuffer_obj(ringbuf); - kfree(ringbuf); + intel_ringbuffer_free(ringbuf); drm_gem_object_unreference(&ctx_obj->base); } } @@ -1867,12 +2410,13 @@ static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring, struct drm_i915_gem_object *default_ctx_obj) { struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct page *page; - /* The status page is offset 0 from the default context object - * in LRC mode. */ - ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj); - ring->status_page.page_addr = - kmap(sg_page(default_ctx_obj->pages->sgl)); + /* The HWSP is part of the default context object in LRC mode. */ + ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj) + + LRC_PPHWSP_PN * PAGE_SIZE; + page = i915_gem_object_get_page(default_ctx_obj, LRC_PPHWSP_PN); + ring->status_page.page_addr = kmap(page); ring->status_page.obj = default_ctx_obj; I915_WRITE(RING_HWS_PGA(ring->mmio_base), @@ -1881,7 +2425,7 @@ static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring, } /** - * intel_lr_context_deferred_create() - create the LRC specific bits of a context + * intel_lr_context_deferred_alloc() - create the LRC specific bits of a context * @ctx: LR context to create. * @ring: engine to be used with the context. * @@ -1893,10 +2437,10 @@ static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring, * * Return: non-zero on error. */ -int intel_lr_context_deferred_create(struct intel_context *ctx, + +int intel_lr_context_deferred_alloc(struct intel_context *ctx, struct intel_engine_cs *ring) { - const bool is_global_default_ctx = (ctx == ring->default_context); struct drm_device *dev = ring->dev; struct drm_i915_gem_object *ctx_obj; uint32_t context_size; @@ -1908,98 +2452,58 @@ int intel_lr_context_deferred_create(struct intel_context *ctx, context_size = round_up(get_lr_context_size(ring), 4096); + /* One extra page as the sharing data between driver and GuC */ + context_size += PAGE_SIZE * LRC_PPHWSP_PN; + ctx_obj = i915_gem_alloc_object(dev, context_size); if (!ctx_obj) { DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); return -ENOMEM; } - if (is_global_default_ctx) { - ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0); - if (ret) { - DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n", - ret); - drm_gem_object_unreference(&ctx_obj->base); - return ret; - } - } - - ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); - if (!ringbuf) { - DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n", - ring->name); - ret = -ENOMEM; - goto error_unpin_ctx; - } - - ringbuf->ring = ring; - - ringbuf->size = 32 * PAGE_SIZE; - ringbuf->effective_size = ringbuf->size; - ringbuf->head = 0; - ringbuf->tail = 0; - ringbuf->last_retired_head = -1; - intel_ring_update_space(ringbuf); - - if (ringbuf->obj == NULL) { - ret = intel_alloc_ringbuffer_obj(dev, ringbuf); - if (ret) { - DRM_DEBUG_DRIVER( - "Failed to allocate ringbuffer obj %s: %d\n", - ring->name, ret); - goto error_free_rbuf; - } - - if (is_global_default_ctx) { - ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf); - if (ret) { - DRM_ERROR( - "Failed to pin and map ringbuffer %s: %d\n", - ring->name, ret); - goto error_destroy_rbuf; - } - } - + ringbuf = intel_engine_create_ringbuffer(ring, 4 * PAGE_SIZE); + if (IS_ERR(ringbuf)) { + ret = PTR_ERR(ringbuf); + goto error_deref_obj; } ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf); if (ret) { DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret); - goto error; + goto error_ringbuf; } ctx->engine[ring->id].ringbuf = ringbuf; ctx->engine[ring->id].state = ctx_obj; - if (ctx == ring->default_context) - lrc_setup_hardware_status_page(ring, ctx_obj); - else if (ring->id == RCS && !ctx->rcs_initialized) { - if (ring->init_context) { - ret = ring->init_context(ring, ctx); - if (ret) { - DRM_ERROR("ring init context: %d\n", ret); - ctx->engine[ring->id].ringbuf = NULL; - ctx->engine[ring->id].state = NULL; - goto error; - } + if (ctx != ring->default_context && ring->init_context) { + struct drm_i915_gem_request *req; + + ret = i915_gem_request_alloc(ring, + ctx, &req); + if (ret) { + DRM_ERROR("ring create req: %d\n", + ret); + goto error_ringbuf; } - ctx->rcs_initialized = true; + ret = ring->init_context(req); + if (ret) { + DRM_ERROR("ring init context: %d\n", + ret); + i915_gem_request_cancel(req); + goto error_ringbuf; + } + i915_add_request_no_flush(req); } - return 0; -error: - if (is_global_default_ctx) - intel_unpin_ringbuffer_obj(ringbuf); -error_destroy_rbuf: - intel_destroy_ringbuffer_obj(ringbuf); -error_free_rbuf: - kfree(ringbuf); -error_unpin_ctx: - if (is_global_default_ctx) - i915_gem_object_ggtt_unpin(ctx_obj); +error_ringbuf: + intel_ringbuffer_free(ringbuf); +error_deref_obj: drm_gem_object_unreference(&ctx_obj->base); + ctx->engine[ring->id].ringbuf = NULL; + ctx->engine[ring->id].state = NULL; return ret; } @@ -2025,7 +2529,7 @@ void intel_lr_context_reset(struct drm_device *dev, WARN(1, "Failed get_pages for context obj\n"); continue; } - page = i915_gem_object_get_page(ctx_obj, 1); + page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); reg_state = kmap_atomic(page); reg_state[CTX_RING_HEAD+1] = 0; diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h index 04d3a6d8b207..0b821b91723a 100644 --- a/drivers/gpu/drm/i915/intel_lrc.h +++ b/drivers/gpu/drm/i915/intel_lrc.h @@ -25,25 +25,30 @@ #define _INTEL_LRC_H_ #define GEN8_LR_CONTEXT_ALIGN 4096 +#define GEN8_CSB_ENTRIES 6 +#define GEN8_CSB_PTR_MASK 0x07 /* Execlists regs */ -#define RING_ELSP(ring) ((ring)->mmio_base+0x230) -#define RING_EXECLIST_STATUS(ring) ((ring)->mmio_base+0x234) -#define RING_CONTEXT_CONTROL(ring) ((ring)->mmio_base+0x244) +#define RING_ELSP(ring) _MMIO((ring)->mmio_base + 0x230) +#define RING_EXECLIST_STATUS_LO(ring) _MMIO((ring)->mmio_base + 0x234) +#define RING_EXECLIST_STATUS_HI(ring) _MMIO((ring)->mmio_base + 0x234 + 4) +#define RING_CONTEXT_CONTROL(ring) _MMIO((ring)->mmio_base + 0x244) #define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH (1 << 3) #define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT (1 << 0) -#define RING_CONTEXT_STATUS_BUF(ring) ((ring)->mmio_base+0x370) -#define RING_CONTEXT_STATUS_PTR(ring) ((ring)->mmio_base+0x3a0) +#define CTX_CTRL_RS_CTX_ENABLE (1 << 1) +#define RING_CONTEXT_STATUS_BUF_LO(ring, i) _MMIO((ring)->mmio_base + 0x370 + (i) * 8) +#define RING_CONTEXT_STATUS_BUF_HI(ring, i) _MMIO((ring)->mmio_base + 0x370 + (i) * 8 + 4) +#define RING_CONTEXT_STATUS_PTR(ring) _MMIO((ring)->mmio_base + 0x3a0) /* Logical Rings */ -int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request, - struct intel_context *ctx); +int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request); +int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request); void intel_logical_ring_stop(struct intel_engine_cs *ring); void intel_logical_ring_cleanup(struct intel_engine_cs *ring); int intel_logical_rings_init(struct drm_device *dev); +int intel_logical_ring_begin(struct drm_i915_gem_request *req, int num_dwords); -int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx); +int logical_ring_flush_all_caches(struct drm_i915_gem_request *req); /** * intel_logical_ring_advance() - advance the ringbuffer tail * @ringbuf: Ringbuffer to advance. @@ -65,25 +70,34 @@ static inline void intel_logical_ring_emit(struct intel_ringbuffer *ringbuf, iowrite32(data, ringbuf->virtual_start + ringbuf->tail); ringbuf->tail += 4; } +static inline void intel_logical_ring_emit_reg(struct intel_ringbuffer *ringbuf, + i915_reg_t reg) +{ + intel_logical_ring_emit(ringbuf, i915_mmio_reg_offset(reg)); +} /* Logical Ring Contexts */ + +/* One extra page is added before LRC for GuC as shared data */ +#define LRC_GUCSHR_PN (0) +#define LRC_PPHWSP_PN (LRC_GUCSHR_PN + 1) +#define LRC_STATE_PN (LRC_PPHWSP_PN + 1) + void intel_lr_context_free(struct intel_context *ctx); -int intel_lr_context_deferred_create(struct intel_context *ctx, - struct intel_engine_cs *ring); -void intel_lr_context_unpin(struct intel_engine_cs *ring, - struct intel_context *ctx); +int intel_lr_context_deferred_alloc(struct intel_context *ctx, + struct intel_engine_cs *ring); +void intel_lr_context_unpin(struct drm_i915_gem_request *req); void intel_lr_context_reset(struct drm_device *dev, struct intel_context *ctx); +uint64_t intel_lr_context_descriptor(struct intel_context *ctx, + struct intel_engine_cs *ring); /* Execlists */ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists); -int intel_execlists_submission(struct drm_device *dev, struct drm_file *file, - struct intel_engine_cs *ring, - struct intel_context *ctx, +struct i915_execbuffer_params; +int intel_execlists_submission(struct i915_execbuffer_params *params, struct drm_i915_gem_execbuffer2 *args, - struct list_head *vmas, - struct drm_i915_gem_object *batch_obj, - u64 exec_start, u32 dispatch_flags); + struct list_head *vmas); u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj); void intel_lrc_irq_handler(struct intel_engine_cs *ring); diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 161ab26f81fb..0da0240caf81 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -51,7 +51,7 @@ struct intel_lvds_encoder { struct intel_encoder base; bool is_dual_link; - u32 reg; + i915_reg_t reg; u32 a3_power; struct intel_lvds_connector *attached_connector; @@ -98,15 +98,11 @@ static void intel_lvds_get_config(struct intel_encoder *encoder, { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 lvds_reg, tmp, flags = 0; + struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); + u32 tmp, flags = 0; int dotclock; - if (HAS_PCH_SPLIT(dev)) - lvds_reg = PCH_LVDS; - else - lvds_reg = LVDS; - - tmp = I915_READ(lvds_reg); + tmp = I915_READ(lvds_encoder->reg); if (tmp & LVDS_HSYNC_POLARITY) flags |= DRM_MODE_FLAG_NHSYNC; else @@ -139,8 +135,7 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder) struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - const struct drm_display_mode *adjusted_mode = - &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; int pipe = crtc->pipe; u32 temp; @@ -215,7 +210,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder) struct intel_connector *intel_connector = &lvds_encoder->attached_connector->base; struct drm_i915_private *dev_priv = dev->dev_private; - u32 ctl_reg, stat_reg; + i915_reg_t ctl_reg, stat_reg; if (HAS_PCH_SPLIT(dev)) { ctl_reg = PCH_PP_CONTROL; @@ -239,10 +234,8 @@ static void intel_disable_lvds(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); - struct intel_connector *intel_connector = - &lvds_encoder->attached_connector->base; struct drm_i915_private *dev_priv = dev->dev_private; - u32 ctl_reg, stat_reg; + i915_reg_t ctl_reg, stat_reg; if (HAS_PCH_SPLIT(dev)) { ctl_reg = PCH_PP_CONTROL; @@ -252,8 +245,6 @@ static void intel_disable_lvds(struct intel_encoder *encoder) stat_reg = PP_STATUS; } - intel_panel_disable_backlight(intel_connector); - I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON); if (wait_for((I915_READ(stat_reg) & PP_ON) == 0, 1000)) DRM_ERROR("timed out waiting for panel to power off\n"); @@ -262,17 +253,45 @@ static void intel_disable_lvds(struct intel_encoder *encoder) POSTING_READ(lvds_encoder->reg); } +static void gmch_disable_lvds(struct intel_encoder *encoder) +{ + struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); + struct intel_connector *intel_connector = + &lvds_encoder->attached_connector->base; + + intel_panel_disable_backlight(intel_connector); + + intel_disable_lvds(encoder); +} + +static void pch_disable_lvds(struct intel_encoder *encoder) +{ + struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); + struct intel_connector *intel_connector = + &lvds_encoder->attached_connector->base; + + intel_panel_disable_backlight(intel_connector); +} + +static void pch_post_disable_lvds(struct intel_encoder *encoder) +{ + intel_disable_lvds(encoder); +} + static enum drm_mode_status intel_lvds_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; + int max_pixclk = to_i915(connector->dev)->max_dotclk_freq; if (mode->hdisplay > fixed_mode->hdisplay) return MODE_PANEL; if (mode->vdisplay > fixed_mode->vdisplay) return MODE_PANEL; + if (fixed_mode->clock > max_pixclk) + return MODE_CLOCK_HIGH; return MODE_OK; } @@ -452,7 +471,7 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val, */ if (!HAS_PCH_SPLIT(dev)) { drm_modeset_lock_all(dev); - intel_modeset_setup_hw_state(dev, true); + intel_display_resume(dev); drm_modeset_unlock_all(dev); } @@ -528,7 +547,7 @@ static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs }; static const struct drm_connector_funcs intel_lvds_connector_funcs = { - .dpms = intel_connector_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_lvds_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_lvds_set_property, @@ -920,6 +939,7 @@ void intel_lvds_init(struct drm_device *dev) struct drm_display_mode *downclock_mode = NULL; struct edid *edid; struct drm_crtc *crtc; + i915_reg_t lvds_reg; u32 lvds; int pipe; u8 pin; @@ -931,7 +951,7 @@ void intel_lvds_init(struct drm_device *dev) if (HAS_PCH_SPLIT(dev)) { I915_WRITE(PCH_PP_CONTROL, I915_READ(PCH_PP_CONTROL) | PANEL_UNLOCK_REGS); - } else { + } else if (INTEL_INFO(dev_priv)->gen < 5) { I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS); } @@ -942,14 +962,15 @@ void intel_lvds_init(struct drm_device *dev) if (dmi_check_system(intel_no_lvds)) return; - pin = GMBUS_PIN_PANEL; - if (!lvds_is_present_in_vbt(dev, &pin)) { - DRM_DEBUG_KMS("LVDS is not present in VBT\n"); - return; - } + if (HAS_PCH_SPLIT(dev)) + lvds_reg = PCH_LVDS; + else + lvds_reg = LVDS; + + lvds = I915_READ(lvds_reg); if (HAS_PCH_SPLIT(dev)) { - if ((I915_READ(PCH_LVDS) & LVDS_DETECTED) == 0) + if ((lvds & LVDS_DETECTED) == 0) return; if (dev_priv->vbt.edp_support) { DRM_DEBUG_KMS("disable LVDS for eDP support\n"); @@ -957,6 +978,27 @@ void intel_lvds_init(struct drm_device *dev) } } + pin = GMBUS_PIN_PANEL; + if (!lvds_is_present_in_vbt(dev, &pin)) { + if ((lvds & LVDS_PORT_EN) == 0) { + DRM_DEBUG_KMS("LVDS is not present in VBT\n"); + return; + } + DRM_DEBUG_KMS("LVDS is not present in VBT, but enabled anyway\n"); + } + + /* Set the Panel Power On/Off timings if uninitialized. */ + if (INTEL_INFO(dev_priv)->gen < 5 && + I915_READ(PP_ON_DELAYS) == 0 && I915_READ(PP_OFF_DELAYS) == 0) { + /* Set T2 to 40ms and T5 to 200ms */ + I915_WRITE(PP_ON_DELAYS, 0x019007d0); + + /* Set T3 to 35ms and Tx to 200ms */ + I915_WRITE(PP_OFF_DELAYS, 0x015e07d0); + + DRM_DEBUG_KMS("Panel power timings uninitialized, setting defaults\n"); + } + lvds_encoder = kzalloc(sizeof(*lvds_encoder), GFP_KERNEL); if (!lvds_encoder) return; @@ -983,12 +1025,17 @@ void intel_lvds_init(struct drm_device *dev) DRM_MODE_CONNECTOR_LVDS); drm_encoder_init(dev, &intel_encoder->base, &intel_lvds_enc_funcs, - DRM_MODE_ENCODER_LVDS); + DRM_MODE_ENCODER_LVDS, NULL); intel_encoder->enable = intel_enable_lvds; intel_encoder->pre_enable = intel_pre_enable_lvds; intel_encoder->compute_config = intel_lvds_compute_config; - intel_encoder->disable = intel_disable_lvds; + if (HAS_PCH_SPLIT(dev_priv)) { + intel_encoder->disable = pch_disable_lvds; + intel_encoder->post_disable = pch_post_disable_lvds; + } else { + intel_encoder->disable = gmch_disable_lvds; + } intel_encoder->get_hw_state = intel_lvds_get_hw_state; intel_encoder->get_config = intel_lvds_get_config; intel_connector->get_hw_state = intel_connector_get_hw_state; @@ -1010,11 +1057,7 @@ void intel_lvds_init(struct drm_device *dev) connector->interlace_allowed = false; connector->doublescan_allowed = false; - if (HAS_PCH_SPLIT(dev)) { - lvds_encoder->reg = PCH_LVDS; - } else { - lvds_encoder->reg = LVDS; - } + lvds_encoder->reg = lvds_reg; /* create the scaling mode property */ drm_mode_create_scaling_mode_property(dev); @@ -1068,24 +1111,8 @@ void intel_lvds_init(struct drm_device *dev) drm_mode_debug_printmodeline(scan); fixed_mode = drm_mode_duplicate(dev, scan); - if (fixed_mode) { - downclock_mode = - intel_find_panel_downclock(dev, - fixed_mode, connector); - if (downclock_mode != NULL && - i915.lvds_downclock) { - /* We found the downclock for LVDS. */ - dev_priv->lvds_downclock_avail = true; - dev_priv->lvds_downclock = - downclock_mode->clock; - DRM_DEBUG_KMS("LVDS downclock is found" - " in EDID. Normal clock %dKhz, " - "downclock %dKhz\n", - fixed_mode->clock, - dev_priv->lvds_downclock); - } + if (fixed_mode) goto out; - } } } @@ -1111,7 +1138,6 @@ void intel_lvds_init(struct drm_device *dev) if (HAS_PCH_SPLIT(dev)) goto failed; - lvds = I915_READ(LVDS); pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0; crtc = intel_get_crtc_for_pipe(dev, pipe); @@ -1138,8 +1164,7 @@ out: DRM_DEBUG_KMS("detected %s-link lvds configuration\n", lvds_encoder->is_dual_link ? "dual" : "single"); - lvds_encoder->a3_power = I915_READ(lvds_encoder->reg) & - LVDS_A3_POWER_MASK; + lvds_encoder->a3_power = lvds & LVDS_A3_POWER_MASK; lvds_connector->lid_notifier.notifier_call = intel_lid_notify; if (acpi_lid_notifier_register(&lvds_connector->lid_notifier)) { diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c new file mode 100644 index 000000000000..fed7bea19cc9 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_mocs.c @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "intel_mocs.h" +#include "intel_lrc.h" +#include "intel_ringbuffer.h" + +/* structures required */ +struct drm_i915_mocs_entry { + u32 control_value; + u16 l3cc_value; +}; + +struct drm_i915_mocs_table { + u32 size; + const struct drm_i915_mocs_entry *table; +}; + +/* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */ +#define LE_CACHEABILITY(value) ((value) << 0) +#define LE_TGT_CACHE(value) ((value) << 2) +#define LE_LRUM(value) ((value) << 4) +#define LE_AOM(value) ((value) << 6) +#define LE_RSC(value) ((value) << 7) +#define LE_SCC(value) ((value) << 8) +#define LE_PFM(value) ((value) << 11) +#define LE_SCF(value) ((value) << 14) + +/* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */ +#define L3_ESC(value) ((value) << 0) +#define L3_SCC(value) ((value) << 1) +#define L3_CACHEABILITY(value) ((value) << 4) + +/* Helper defines */ +#define GEN9_NUM_MOCS_ENTRIES 62 /* 62 out of 64 - 63 & 64 are reserved. */ + +/* (e)LLC caching options */ +#define LE_PAGETABLE 0 +#define LE_UC 1 +#define LE_WT 2 +#define LE_WB 3 + +/* L3 caching options */ +#define L3_DIRECT 0 +#define L3_UC 1 +#define L3_RESERVED 2 +#define L3_WB 3 + +/* Target cache */ +#define ELLC 0 +#define LLC 1 +#define LLC_ELLC 2 + +/* + * MOCS tables + * + * These are the MOCS tables that are programmed across all the rings. + * The control value is programmed to all the rings that support the + * MOCS registers. While the l3cc_values are only programmed to the + * LNCFCMOCS0 - LNCFCMOCS32 registers. + * + * These tables are intended to be kept reasonably consistent across + * platforms. However some of the fields are not applicable to all of + * them. + * + * Entries not part of the following tables are undefined as far as + * userspace is concerned and shouldn't be relied upon. For the time + * being they will be implicitly initialized to the strictest caching + * configuration (uncached) to guarantee forwards compatibility with + * userspace programs written against more recent kernels providing + * additional MOCS entries. + * + * NOTE: These tables MUST start with being uncached and the length + * MUST be less than 63 as the last two registers are reserved + * by the hardware. These tables are part of the kernel ABI and + * may only be updated incrementally by adding entries at the + * end. + */ +static const struct drm_i915_mocs_entry skylake_mocs_table[] = { + /* { 0x00000009, 0x0010 } */ + { (LE_CACHEABILITY(LE_UC) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(0) | + LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), + (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC)) }, + /* { 0x00000038, 0x0030 } */ + { (LE_CACHEABILITY(LE_PAGETABLE) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | + LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), + (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) }, + /* { 0x0000003b, 0x0030 } */ + { (LE_CACHEABILITY(LE_WB) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | + LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), + (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) } +}; + +/* NOTE: the LE_TGT_CACHE is not used on Broxton */ +static const struct drm_i915_mocs_entry broxton_mocs_table[] = { + /* { 0x00000009, 0x0010 } */ + { (LE_CACHEABILITY(LE_UC) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(0) | + LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), + (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC)) }, + /* { 0x00000038, 0x0030 } */ + { (LE_CACHEABILITY(LE_PAGETABLE) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | + LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), + (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) }, + /* { 0x0000003b, 0x0030 } */ + { (LE_CACHEABILITY(LE_WB) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | + LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), + (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) } +}; + +/** + * get_mocs_settings() + * @dev: DRM device. + * @table: Output table that will be made to point at appropriate + * MOCS values for the device. + * + * This function will return the values of the MOCS table that needs to + * be programmed for the platform. It will return the values that need + * to be programmed and if they need to be programmed. + * + * Return: true if there are applicable MOCS settings for the device. + */ +static bool get_mocs_settings(struct drm_device *dev, + struct drm_i915_mocs_table *table) +{ + bool result = false; + + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + table->size = ARRAY_SIZE(skylake_mocs_table); + table->table = skylake_mocs_table; + result = true; + } else if (IS_BROXTON(dev)) { + table->size = ARRAY_SIZE(broxton_mocs_table); + table->table = broxton_mocs_table; + result = true; + } else { + WARN_ONCE(INTEL_INFO(dev)->gen >= 9, + "Platform that should have a MOCS table does not.\n"); + } + + return result; +} + +static i915_reg_t mocs_register(enum intel_ring_id ring, int index) +{ + switch (ring) { + case RCS: + return GEN9_GFX_MOCS(index); + case VCS: + return GEN9_MFX0_MOCS(index); + case BCS: + return GEN9_BLT_MOCS(index); + case VECS: + return GEN9_VEBOX_MOCS(index); + case VCS2: + return GEN9_MFX1_MOCS(index); + default: + MISSING_CASE(ring); + return INVALID_MMIO_REG; + } +} + +/** + * emit_mocs_control_table() - emit the mocs control table + * @req: Request to set up the MOCS table for. + * @table: The values to program into the control regs. + * @ring: The engine for whom to emit the registers. + * + * This function simply emits a MI_LOAD_REGISTER_IMM command for the + * given table starting at the given address. + * + * Return: 0 on success, otherwise the error status. + */ +static int emit_mocs_control_table(struct drm_i915_gem_request *req, + const struct drm_i915_mocs_table *table, + enum intel_ring_id ring) +{ + struct intel_ringbuffer *ringbuf = req->ringbuf; + unsigned int index; + int ret; + + if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) + return -ENODEV; + + ret = intel_logical_ring_begin(req, 2 + 2 * GEN9_NUM_MOCS_ENTRIES); + if (ret) { + DRM_DEBUG("intel_logical_ring_begin failed %d\n", ret); + return ret; + } + + intel_logical_ring_emit(ringbuf, + MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES)); + + for (index = 0; index < table->size; index++) { + intel_logical_ring_emit_reg(ringbuf, mocs_register(ring, index)); + intel_logical_ring_emit(ringbuf, + table->table[index].control_value); + } + + /* + * Ok, now set the unused entries to uncached. These entries + * are officially undefined and no contract for the contents + * and settings is given for these entries. + * + * Entry 0 in the table is uncached - so we are just writing + * that value to all the used entries. + */ + for (; index < GEN9_NUM_MOCS_ENTRIES; index++) { + intel_logical_ring_emit_reg(ringbuf, mocs_register(ring, index)); + intel_logical_ring_emit(ringbuf, table->table[0].control_value); + } + + intel_logical_ring_emit(ringbuf, MI_NOOP); + intel_logical_ring_advance(ringbuf); + + return 0; +} + +/** + * emit_mocs_l3cc_table() - emit the mocs control table + * @req: Request to set up the MOCS table for. + * @table: The values to program into the control regs. + * + * This function simply emits a MI_LOAD_REGISTER_IMM command for the + * given table starting at the given address. This register set is + * programmed in pairs. + * + * Return: 0 on success, otherwise the error status. + */ +static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req, + const struct drm_i915_mocs_table *table) +{ + struct intel_ringbuffer *ringbuf = req->ringbuf; + unsigned int count; + unsigned int i; + u32 value; + u32 filler = (table->table[0].l3cc_value & 0xffff) | + ((table->table[0].l3cc_value & 0xffff) << 16); + int ret; + + if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) + return -ENODEV; + + ret = intel_logical_ring_begin(req, 2 + GEN9_NUM_MOCS_ENTRIES); + if (ret) { + DRM_DEBUG("intel_logical_ring_begin failed %d\n", ret); + return ret; + } + + intel_logical_ring_emit(ringbuf, + MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES / 2)); + + for (i = 0, count = 0; i < table->size / 2; i++, count += 2) { + value = (table->table[count].l3cc_value & 0xffff) | + ((table->table[count + 1].l3cc_value & 0xffff) << 16); + + intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i)); + intel_logical_ring_emit(ringbuf, value); + } + + if (table->size & 0x01) { + /* Odd table size - 1 left over */ + value = (table->table[count].l3cc_value & 0xffff) | + ((table->table[0].l3cc_value & 0xffff) << 16); + } else + value = filler; + + /* + * Now set the rest of the table to uncached - use entry 0 as + * this will be uncached. Leave the last pair uninitialised as + * they are reserved by the hardware. + */ + for (; i < GEN9_NUM_MOCS_ENTRIES / 2; i++) { + intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i)); + intel_logical_ring_emit(ringbuf, value); + + value = filler; + } + + intel_logical_ring_emit(ringbuf, MI_NOOP); + intel_logical_ring_advance(ringbuf); + + return 0; +} + +/** + * intel_rcs_context_init_mocs() - program the MOCS register. + * @req: Request to set up the MOCS tables for. + * + * This function will emit a batch buffer with the values required for + * programming the MOCS register values for all the currently supported + * rings. + * + * These registers are partially stored in the RCS context, so they are + * emitted at the same time so that when a context is created these registers + * are set up. These registers have to be emitted into the start of the + * context as setting the ELSP will re-init some of these registers back + * to the hw values. + * + * Return: 0 on success, otherwise the error status. + */ +int intel_rcs_context_init_mocs(struct drm_i915_gem_request *req) +{ + struct drm_i915_mocs_table t; + int ret; + + if (get_mocs_settings(req->ring->dev, &t)) { + struct drm_i915_private *dev_priv = req->i915; + struct intel_engine_cs *ring; + enum intel_ring_id ring_id; + + /* Program the control registers */ + for_each_ring(ring, dev_priv, ring_id) { + ret = emit_mocs_control_table(req, &t, ring_id); + if (ret) + return ret; + } + + /* Now program the l3cc registers */ + ret = emit_mocs_l3cc_table(req, &t); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/gpu/drm/i915/intel_mocs.h b/drivers/gpu/drm/i915/intel_mocs.h new file mode 100644 index 000000000000..76e45b1748b3 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_mocs.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INTEL_MOCS_H +#define INTEL_MOCS_H + +/** + * DOC: Memory Objects Control State (MOCS) + * + * Motivation: + * In previous Gens the MOCS settings was a value that was set by user land as + * part of the batch. In Gen9 this has changed to be a single table (per ring) + * that all batches now reference by index instead of programming the MOCS + * directly. + * + * The one wrinkle in this is that only PART of the MOCS tables are included + * in context (The GFX_MOCS_0 - GFX_MOCS_64 and the LNCFCMOCS0 - LNCFCMOCS32 + * registers). The rest are not (the settings for the other rings). + * + * This table needs to be set at system start-up because the way the table + * interacts with the contexts and the GmmLib interface. + * + * + * Implementation: + * + * The tables (one per supported platform) are defined in intel_mocs.c + * and are programmed in the first batch after the context is loaded + * (with the hardware workarounds). This will then let the usual + * context handling keep the MOCS in step. + */ + +#include <drm/drmP.h> +#include "i915_drv.h" + +int intel_rcs_context_init_mocs(struct drm_i915_gem_request *req); + +#endif diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c index 0e860f39933d..38a4c8ce7e63 100644 --- a/drivers/gpu/drm/i915/intel_modes.c +++ b/drivers/gpu/drm/i915/intel_modes.c @@ -126,3 +126,12 @@ intel_attach_broadcast_rgb_property(struct drm_connector *connector) drm_object_attach_property(&connector->base, prop, 0); } + +void +intel_attach_aspect_ratio_property(struct drm_connector *connector) +{ + if (!drm_mode_create_aspect_ratio_property(connector->dev)) + drm_object_attach_property(&connector->base, + connector->dev->mode_config.aspect_ratio_property, + DRM_MODE_PICTURE_ASPECT_NONE); +} diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 481337436f72..e362a30776fa 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -25,8 +25,6 @@ * */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include <linux/acpi.h> #include <acpi/video.h> @@ -53,6 +51,7 @@ #define MBOX_ACPI (1<<0) #define MBOX_SWSCI (1<<1) #define MBOX_ASLE (1<<2) +#define MBOX_ASLE_EXT (1<<4) struct opregion_header { u8 signature[16]; @@ -62,7 +61,10 @@ struct opregion_header { u8 vbios_ver[16]; u8 driver_ver[16]; u32 mboxes; - u8 reserved[164]; + u32 driver_model; + u32 pcon; + u8 dver[32]; + u8 rsvd[124]; } __packed; /* OpRegion mailbox #1: public ACPI methods */ @@ -84,7 +86,9 @@ struct opregion_acpi { u32 evts; /* ASL supported events */ u32 cnot; /* current OS notification */ u32 nrdy; /* driver status */ - u8 rsvd2[60]; + u32 did2[7]; /* extended supported display devices ID list */ + u32 cpd2[7]; /* extended attached display devices list */ + u8 rsvd2[4]; } __packed; /* OpRegion mailbox #2: SWSCI */ @@ -113,7 +117,10 @@ struct opregion_asle { u32 pcft; /* power conservation features */ u32 srot; /* supported rotation angles */ u32 iuer; /* IUER events */ - u8 rsvd[86]; + u64 fdss; + u32 fdsp; + u32 stat; + u8 rsvd[70]; } __packed; /* Driver readiness indicator */ @@ -232,7 +239,7 @@ struct opregion_asle { static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) { struct drm_i915_private *dev_priv = dev->dev_private; - struct opregion_swsci __iomem *swsci = dev_priv->opregion.swsci; + struct opregion_swsci *swsci = dev_priv->opregion.swsci; u32 main_function, sub_function, scic; u16 pci_swsci; u32 dslp; @@ -257,7 +264,7 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) } /* Driver sleep timeout in ms. */ - dslp = ioread32(&swsci->dslp); + dslp = swsci->dslp; if (!dslp) { /* The spec says 2ms should be the default, but it's too small * for some machines. */ @@ -270,7 +277,7 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) } /* The spec tells us to do this, but we are the only user... */ - scic = ioread32(&swsci->scic); + scic = swsci->scic; if (scic & SWSCI_SCIC_INDICATOR) { DRM_DEBUG_DRIVER("SWSCI request already in progress\n"); return -EBUSY; @@ -278,8 +285,8 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) scic = function | SWSCI_SCIC_INDICATOR; - iowrite32(parm, &swsci->parm); - iowrite32(scic, &swsci->scic); + swsci->parm = parm; + swsci->scic = scic; /* Ensure SCI event is selected and event trigger is cleared. */ pci_read_config_word(dev->pdev, PCI_SWSCI, &pci_swsci); @@ -294,7 +301,7 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) pci_write_config_word(dev->pdev, PCI_SWSCI, pci_swsci); /* Poll for the result. */ -#define C (((scic = ioread32(&swsci->scic)) & SWSCI_SCIC_INDICATOR) == 0) +#define C (((scic = swsci->scic) & SWSCI_SCIC_INDICATOR) == 0) if (wait_for(C, dslp)) { DRM_DEBUG_DRIVER("SWSCI request timed out\n"); return -ETIMEDOUT; @@ -310,7 +317,7 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) } if (parm_out) - *parm_out = ioread32(&swsci->parm); + *parm_out = swsci->parm; return 0; @@ -334,8 +341,12 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, if (!HAS_DDI(dev)) return 0; - port = intel_ddi_get_encoder_port(intel_encoder); - if (port == PORT_E) { + if (intel_encoder->type == INTEL_OUTPUT_DSI) + port = 0; + else + port = intel_ddi_get_encoder_port(intel_encoder); + + if (port == PORT_E) { port = 0; } else { parm |= 1 << port; @@ -356,6 +367,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, type = DISPLAY_TYPE_EXTERNAL_FLAT_PANEL; break; case INTEL_OUTPUT_EDP: + case INTEL_OUTPUT_DSI: type = DISPLAY_TYPE_INTERNAL_FLAT_PANEL; break; default: @@ -400,7 +412,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_connector *intel_connector; - struct opregion_asle __iomem *asle = dev_priv->opregion.asle; + struct opregion_asle *asle = dev_priv->opregion.asle; DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp); @@ -425,7 +437,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) DRM_DEBUG_KMS("updating opregion backlight %d/255\n", bclp); list_for_each_entry(intel_connector, &dev->mode_config.connector_list, base.head) intel_panel_set_backlight_acpi(intel_connector, bclp, 255); - iowrite32(DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID, &asle->cblv); + asle->cblv = DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID; drm_modeset_unlock(&dev->mode_config.connection_mutex); @@ -512,14 +524,14 @@ static void asle_work(struct work_struct *work) struct drm_i915_private *dev_priv = container_of(opregion, struct drm_i915_private, opregion); struct drm_device *dev = dev_priv->dev; - struct opregion_asle __iomem *asle = dev_priv->opregion.asle; + struct opregion_asle *asle = dev_priv->opregion.asle; u32 aslc_stat = 0; u32 aslc_req; if (!asle) return; - aslc_req = ioread32(&asle->aslc); + aslc_req = asle->aslc; if (!(aslc_req & ASLC_REQ_MSK)) { DRM_DEBUG_DRIVER("No request on ASLC interrupt 0x%08x\n", @@ -528,34 +540,34 @@ static void asle_work(struct work_struct *work) } if (aslc_req & ASLC_SET_ALS_ILLUM) - aslc_stat |= asle_set_als_illum(dev, ioread32(&asle->alsi)); + aslc_stat |= asle_set_als_illum(dev, asle->alsi); if (aslc_req & ASLC_SET_BACKLIGHT) - aslc_stat |= asle_set_backlight(dev, ioread32(&asle->bclp)); + aslc_stat |= asle_set_backlight(dev, asle->bclp); if (aslc_req & ASLC_SET_PFIT) - aslc_stat |= asle_set_pfit(dev, ioread32(&asle->pfit)); + aslc_stat |= asle_set_pfit(dev, asle->pfit); if (aslc_req & ASLC_SET_PWM_FREQ) - aslc_stat |= asle_set_pwm_freq(dev, ioread32(&asle->pfmb)); + aslc_stat |= asle_set_pwm_freq(dev, asle->pfmb); if (aslc_req & ASLC_SUPPORTED_ROTATION_ANGLES) aslc_stat |= asle_set_supported_rotation_angles(dev, - ioread32(&asle->srot)); + asle->srot); if (aslc_req & ASLC_BUTTON_ARRAY) - aslc_stat |= asle_set_button_array(dev, ioread32(&asle->iuer)); + aslc_stat |= asle_set_button_array(dev, asle->iuer); if (aslc_req & ASLC_CONVERTIBLE_INDICATOR) - aslc_stat |= asle_set_convertible(dev, ioread32(&asle->iuer)); + aslc_stat |= asle_set_convertible(dev, asle->iuer); if (aslc_req & ASLC_DOCKING_INDICATOR) - aslc_stat |= asle_set_docking(dev, ioread32(&asle->iuer)); + aslc_stat |= asle_set_docking(dev, asle->iuer); if (aslc_req & ASLC_ISCT_STATE_CHANGE) aslc_stat |= asle_isct_state(dev); - iowrite32(aslc_stat, &asle->aslc); + asle->aslc = aslc_stat; } void intel_opregion_asle_intr(struct drm_device *dev) @@ -580,8 +592,8 @@ static int intel_opregion_video_event(struct notifier_block *nb, Linux, these are handled by the dock, button and video drivers. */ - struct opregion_acpi __iomem *acpi; struct acpi_bus_event *event = data; + struct opregion_acpi *acpi; int ret = NOTIFY_OK; if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0) @@ -592,11 +604,10 @@ static int intel_opregion_video_event(struct notifier_block *nb, acpi = system_opregion->acpi; - if (event->type == 0x80 && - (ioread32(&acpi->cevt) & 1) == 0) + if (event->type == 0x80 && ((acpi->cevt & 1) == 0)) ret = NOTIFY_BAD; - iowrite32(0, &acpi->csts); + acpi->csts = 0; return ret; } @@ -611,6 +622,38 @@ static struct notifier_block intel_opregion_notifier = { * (version 3) */ +static u32 get_did(struct intel_opregion *opregion, int i) +{ + u32 did; + + if (i < ARRAY_SIZE(opregion->acpi->didl)) { + did = opregion->acpi->didl[i]; + } else { + i -= ARRAY_SIZE(opregion->acpi->didl); + + if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2))) + return 0; + + did = opregion->acpi->did2[i]; + } + + return did; +} + +static void set_did(struct intel_opregion *opregion, int i, u32 val) +{ + if (i < ARRAY_SIZE(opregion->acpi->didl)) { + opregion->acpi->didl[i] = val; + } else { + i -= ARRAY_SIZE(opregion->acpi->didl); + + if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2))) + return; + + opregion->acpi->did2[i] = val; + } +} + static void intel_didl_outputs(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -620,7 +663,7 @@ static void intel_didl_outputs(struct drm_device *dev) struct acpi_device *acpi_dev, *acpi_cdev, *acpi_video_bus = NULL; unsigned long long device_id; acpi_status status; - u32 temp; + u32 temp, max_outputs; int i = 0; handle = ACPI_HANDLE(&dev->pdev->dev); @@ -639,41 +682,50 @@ static void intel_didl_outputs(struct drm_device *dev) } if (!acpi_video_bus) { - pr_warn("No ACPI video bus found\n"); + DRM_DEBUG_KMS("No ACPI video bus found\n"); return; } + /* + * In theory, did2, the extended didl, gets added at opregion version + * 3.0. In practice, however, we're supposed to set it for earlier + * versions as well, since a BIOS that doesn't understand did2 should + * not look at it anyway. Use a variable so we can tweak this if a need + * arises later. + */ + max_outputs = ARRAY_SIZE(opregion->acpi->didl) + + ARRAY_SIZE(opregion->acpi->did2); + list_for_each_entry(acpi_cdev, &acpi_video_bus->children, node) { - if (i >= 8) { - dev_dbg(&dev->pdev->dev, - "More than 8 outputs detected via ACPI\n"); + if (i >= max_outputs) { + DRM_DEBUG_KMS("More than %u outputs detected via ACPI\n", + max_outputs); return; } - status = - acpi_evaluate_integer(acpi_cdev->handle, "_ADR", - NULL, &device_id); + status = acpi_evaluate_integer(acpi_cdev->handle, "_ADR", + NULL, &device_id); if (ACPI_SUCCESS(status)) { if (!device_id) goto blind_set; - iowrite32((u32)(device_id & 0x0f0f), - &opregion->acpi->didl[i]); - i++; + set_did(opregion, i++, (u32)(device_id & 0x0f0f)); } } end: - /* If fewer than 8 outputs, the list must be null terminated */ - if (i < 8) - iowrite32(0, &opregion->acpi->didl[i]); + DRM_DEBUG_KMS("%d outputs detected\n", i); + + /* If fewer than max outputs, the list must be null terminated */ + if (i < max_outputs) + set_did(opregion, i, 0); return; blind_set: i = 0; list_for_each_entry(connector, &dev->mode_config.connector_list, head) { int output_type = ACPI_OTHER_OUTPUT; - if (i >= 8) { - dev_dbg(&dev->pdev->dev, - "More than 8 outputs in connector list\n"); + if (i >= max_outputs) { + DRM_DEBUG_KMS("More than %u outputs in connector list\n", + max_outputs); return; } switch (connector->connector_type) { @@ -698,9 +750,8 @@ blind_set: output_type = ACPI_LVDS_OUTPUT; break; } - temp = ioread32(&opregion->acpi->didl[i]); - iowrite32(temp | (1<<31) | output_type | i, - &opregion->acpi->didl[i]); + temp = get_did(opregion, i); + set_did(opregion, i, temp | (1 << 31) | output_type | i); i++; } goto end; @@ -720,8 +771,8 @@ static void intel_setup_cadls(struct drm_device *dev) * display switching hotkeys. Just like DIDL, CADL is NULL-terminated if * there are less than eight devices. */ do { - disp_id = ioread32(&opregion->acpi->didl[i]); - iowrite32(disp_id, &opregion->acpi->cadl[i]); + disp_id = get_did(opregion, i); + opregion->acpi->cadl[i] = disp_id; } while (++i < 8 && disp_id != 0); } @@ -740,16 +791,16 @@ void intel_opregion_init(struct drm_device *dev) /* Notify BIOS we are ready to handle ACPI video ext notifs. * Right now, all the events are handled by the ACPI video module. * We don't actually need to do anything with them. */ - iowrite32(0, &opregion->acpi->csts); - iowrite32(1, &opregion->acpi->drdy); + opregion->acpi->csts = 0; + opregion->acpi->drdy = 1; system_opregion = opregion; register_acpi_notifier(&intel_opregion_notifier); } if (opregion->asle) { - iowrite32(ASLE_TCHE_BLC_EN, &opregion->asle->tche); - iowrite32(ASLE_ARDY_READY, &opregion->asle->ardy); + opregion->asle->tche = ASLE_TCHE_BLC_EN; + opregion->asle->ardy = ASLE_ARDY_READY; } } @@ -762,19 +813,19 @@ void intel_opregion_fini(struct drm_device *dev) return; if (opregion->asle) - iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy); + opregion->asle->ardy = ASLE_ARDY_NOT_READY; cancel_work_sync(&dev_priv->opregion.asle_work); if (opregion->acpi) { - iowrite32(0, &opregion->acpi->drdy); + opregion->acpi->drdy = 0; system_opregion = NULL; unregister_acpi_notifier(&intel_opregion_notifier); } /* just clear all opregion memory pointers now */ - iounmap(opregion->header); + memunmap(opregion->header); opregion->header = NULL; opregion->acpi = NULL; opregion->swsci = NULL; @@ -847,10 +898,15 @@ int intel_opregion_setup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; - void __iomem *base; u32 asls, mboxes; char buf[sizeof(OPREGION_SIGNATURE)]; int err = 0; + void *base; + + BUILD_BUG_ON(sizeof(struct opregion_header) != 0x100); + BUILD_BUG_ON(sizeof(struct opregion_acpi) != 0x100); + BUILD_BUG_ON(sizeof(struct opregion_swsci) != 0x100); + BUILD_BUG_ON(sizeof(struct opregion_asle) != 0x100); pci_read_config_dword(dev->pdev, PCI_ASLS, &asls); DRM_DEBUG_DRIVER("graphic opregion physical addr: 0x%x\n", asls); @@ -863,11 +919,11 @@ int intel_opregion_setup(struct drm_device *dev) INIT_WORK(&opregion->asle_work, asle_work); #endif - base = acpi_os_ioremap(asls, OPREGION_SIZE); + base = memremap(asls, OPREGION_SIZE, MEMREMAP_WB); if (!base) return -ENOMEM; - memcpy_fromio(buf, base, sizeof(buf)); + memcpy(buf, base, sizeof(buf)); if (memcmp(buf, OPREGION_SIGNATURE, 16)) { DRM_DEBUG_DRIVER("opregion signature mismatch\n"); @@ -879,7 +935,7 @@ int intel_opregion_setup(struct drm_device *dev) opregion->lid_state = base + ACPI_CLID; - mboxes = ioread32(&opregion->header->mboxes); + mboxes = opregion->header->mboxes; if (mboxes & MBOX_ACPI) { DRM_DEBUG_DRIVER("Public ACPI methods supported\n"); opregion->acpi = base + OPREGION_ACPI_OFFSET; @@ -894,12 +950,12 @@ int intel_opregion_setup(struct drm_device *dev) DRM_DEBUG_DRIVER("ASLE supported\n"); opregion->asle = base + OPREGION_ASLE_OFFSET; - iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy); + opregion->asle->ardy = ASLE_ARDY_NOT_READY; } return 0; err_out: - iounmap(base); + memunmap(base); return err; } diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index 25c8ec697da1..76f1980a7541 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -210,19 +210,14 @@ static void intel_overlay_unmap_regs(struct intel_overlay *overlay, } static int intel_overlay_do_wait_request(struct intel_overlay *overlay, + struct drm_i915_gem_request *req, void (*tail)(struct intel_overlay *)) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring = &dev_priv->ring[RCS]; int ret; WARN_ON(overlay->last_flip_req); - i915_gem_request_assign(&overlay->last_flip_req, - ring->outstanding_lazy_request); - ret = i915_add_request(ring); - if (ret) - return ret; + i915_gem_request_assign(&overlay->last_flip_req, req); + i915_add_request(req); overlay->flip_tail = tail; ret = i915_wait_request(overlay->last_flip_req); @@ -239,15 +234,22 @@ static int intel_overlay_on(struct intel_overlay *overlay) struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring = &dev_priv->ring[RCS]; + struct drm_i915_gem_request *req; int ret; WARN_ON(overlay->active); WARN_ON(IS_I830(dev) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE)); - ret = intel_ring_begin(ring, 4); + ret = i915_gem_request_alloc(ring, ring->default_context, &req); if (ret) return ret; + ret = intel_ring_begin(req, 4); + if (ret) { + i915_gem_request_cancel(req); + return ret; + } + overlay->active = true; intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_ON); @@ -256,7 +258,7 @@ static int intel_overlay_on(struct intel_overlay *overlay) intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); - return intel_overlay_do_wait_request(overlay, NULL); + return intel_overlay_do_wait_request(overlay, req, NULL); } /* overlay needs to be enabled in OCMD reg */ @@ -266,6 +268,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay, struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring = &dev_priv->ring[RCS]; + struct drm_i915_gem_request *req; u32 flip_addr = overlay->flip_addr; u32 tmp; int ret; @@ -280,18 +283,25 @@ static int intel_overlay_continue(struct intel_overlay *overlay, if (tmp & (1 << 17)) DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); - ret = intel_ring_begin(ring, 2); + ret = i915_gem_request_alloc(ring, ring->default_context, &req); if (ret) return ret; + ret = intel_ring_begin(req, 2); + if (ret) { + i915_gem_request_cancel(req); + return ret; + } + intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); intel_ring_emit(ring, flip_addr); intel_ring_advance(ring); WARN_ON(overlay->last_flip_req); - i915_gem_request_assign(&overlay->last_flip_req, - ring->outstanding_lazy_request); - return i915_add_request(ring); + i915_gem_request_assign(&overlay->last_flip_req, req); + i915_add_request(req); + + return 0; } static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay) @@ -327,6 +337,7 @@ static int intel_overlay_off(struct intel_overlay *overlay) struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *ring = &dev_priv->ring[RCS]; + struct drm_i915_gem_request *req; u32 flip_addr = overlay->flip_addr; int ret; @@ -338,10 +349,16 @@ static int intel_overlay_off(struct intel_overlay *overlay) * of the hw. Do it in both cases */ flip_addr |= OFC_UPDATE; - ret = intel_ring_begin(ring, 6); + ret = i915_gem_request_alloc(ring, ring->default_context, &req); if (ret) return ret; + ret = intel_ring_begin(req, 6); + if (ret) { + i915_gem_request_cancel(req); + return ret; + } + /* wait for overlay to go idle */ intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); intel_ring_emit(ring, flip_addr); @@ -360,7 +377,7 @@ static int intel_overlay_off(struct intel_overlay *overlay) } intel_ring_advance(ring); - return intel_overlay_do_wait_request(overlay, intel_overlay_off_tail); + return intel_overlay_do_wait_request(overlay, req, intel_overlay_off_tail); } /* recover from an interruption due to a signal @@ -404,15 +421,23 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay) if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) { /* synchronous slowpath */ - ret = intel_ring_begin(ring, 2); + struct drm_i915_gem_request *req; + + ret = i915_gem_request_alloc(ring, ring->default_context, &req); if (ret) return ret; + ret = intel_ring_begin(req, 2); + if (ret) { + i915_gem_request_cancel(req); + return ret; + } + intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); - ret = intel_overlay_do_wait_request(overlay, + ret = intel_overlay_do_wait_request(overlay, req, intel_overlay_release_old_vid_tail); if (ret) return ret; @@ -724,7 +749,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, if (ret != 0) return ret; - ret = i915_gem_object_pin_to_display_plane(new_bo, 0, NULL, + ret = i915_gem_object_pin_to_display_plane(new_bo, 0, &i915_ggtt_view_normal); if (ret != 0) return ret; diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 55aad2322e10..ae808b68a44f 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -32,8 +32,11 @@ #include <linux/kernel.h> #include <linux/moduleparam.h> +#include <linux/pwm.h> #include "intel_drv.h" +#define CRC_PMIC_PWM_PERIOD_NS 21333 + void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, struct drm_display_mode *adjusted_mode) @@ -102,59 +105,55 @@ intel_pch_panel_fitting(struct intel_crtc *intel_crtc, struct intel_crtc_state *pipe_config, int fitting_mode) { - struct drm_display_mode *adjusted_mode; - int x, y, width, height; - - adjusted_mode = &pipe_config->base.adjusted_mode; - - x = y = width = height = 0; + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + int x = 0, y = 0, width = 0, height = 0; /* Native modes don't need fitting */ - if (adjusted_mode->hdisplay == pipe_config->pipe_src_w && - adjusted_mode->vdisplay == pipe_config->pipe_src_h) + if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w && + adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h) goto done; switch (fitting_mode) { case DRM_MODE_SCALE_CENTER: width = pipe_config->pipe_src_w; height = pipe_config->pipe_src_h; - x = (adjusted_mode->hdisplay - width + 1)/2; - y = (adjusted_mode->vdisplay - height + 1)/2; + x = (adjusted_mode->crtc_hdisplay - width + 1)/2; + y = (adjusted_mode->crtc_vdisplay - height + 1)/2; break; case DRM_MODE_SCALE_ASPECT: /* Scale but preserve the aspect ratio */ { - u32 scaled_width = adjusted_mode->hdisplay + u32 scaled_width = adjusted_mode->crtc_hdisplay * pipe_config->pipe_src_h; u32 scaled_height = pipe_config->pipe_src_w - * adjusted_mode->vdisplay; + * adjusted_mode->crtc_vdisplay; if (scaled_width > scaled_height) { /* pillar */ width = scaled_height / pipe_config->pipe_src_h; if (width & 1) width++; - x = (adjusted_mode->hdisplay - width + 1) / 2; + x = (adjusted_mode->crtc_hdisplay - width + 1) / 2; y = 0; - height = adjusted_mode->vdisplay; + height = adjusted_mode->crtc_vdisplay; } else if (scaled_width < scaled_height) { /* letter */ height = scaled_width / pipe_config->pipe_src_w; if (height & 1) height++; - y = (adjusted_mode->vdisplay - height + 1) / 2; + y = (adjusted_mode->crtc_vdisplay - height + 1) / 2; x = 0; - width = adjusted_mode->hdisplay; + width = adjusted_mode->crtc_hdisplay; } else { x = y = 0; - width = adjusted_mode->hdisplay; - height = adjusted_mode->vdisplay; + width = adjusted_mode->crtc_hdisplay; + height = adjusted_mode->crtc_vdisplay; } } break; case DRM_MODE_SCALE_FULLSCREEN: x = y = 0; - width = adjusted_mode->hdisplay; - height = adjusted_mode->vdisplay; + width = adjusted_mode->crtc_hdisplay; + height = adjusted_mode->crtc_vdisplay; break; default: @@ -169,46 +168,46 @@ done: } static void -centre_horizontally(struct drm_display_mode *mode, +centre_horizontally(struct drm_display_mode *adjusted_mode, int width) { u32 border, sync_pos, blank_width, sync_width; /* keep the hsync and hblank widths constant */ - sync_width = mode->crtc_hsync_end - mode->crtc_hsync_start; - blank_width = mode->crtc_hblank_end - mode->crtc_hblank_start; + sync_width = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; + blank_width = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start; sync_pos = (blank_width - sync_width + 1) / 2; - border = (mode->hdisplay - width + 1) / 2; + border = (adjusted_mode->crtc_hdisplay - width + 1) / 2; border += border & 1; /* make the border even */ - mode->crtc_hdisplay = width; - mode->crtc_hblank_start = width + border; - mode->crtc_hblank_end = mode->crtc_hblank_start + blank_width; + adjusted_mode->crtc_hdisplay = width; + adjusted_mode->crtc_hblank_start = width + border; + adjusted_mode->crtc_hblank_end = adjusted_mode->crtc_hblank_start + blank_width; - mode->crtc_hsync_start = mode->crtc_hblank_start + sync_pos; - mode->crtc_hsync_end = mode->crtc_hsync_start + sync_width; + adjusted_mode->crtc_hsync_start = adjusted_mode->crtc_hblank_start + sync_pos; + adjusted_mode->crtc_hsync_end = adjusted_mode->crtc_hsync_start + sync_width; } static void -centre_vertically(struct drm_display_mode *mode, +centre_vertically(struct drm_display_mode *adjusted_mode, int height) { u32 border, sync_pos, blank_width, sync_width; /* keep the vsync and vblank widths constant */ - sync_width = mode->crtc_vsync_end - mode->crtc_vsync_start; - blank_width = mode->crtc_vblank_end - mode->crtc_vblank_start; + sync_width = adjusted_mode->crtc_vsync_end - adjusted_mode->crtc_vsync_start; + blank_width = adjusted_mode->crtc_vblank_end - adjusted_mode->crtc_vblank_start; sync_pos = (blank_width - sync_width + 1) / 2; - border = (mode->vdisplay - height + 1) / 2; + border = (adjusted_mode->crtc_vdisplay - height + 1) / 2; - mode->crtc_vdisplay = height; - mode->crtc_vblank_start = height + border; - mode->crtc_vblank_end = mode->crtc_vblank_start + blank_width; + adjusted_mode->crtc_vdisplay = height; + adjusted_mode->crtc_vblank_start = height + border; + adjusted_mode->crtc_vblank_end = adjusted_mode->crtc_vblank_start + blank_width; - mode->crtc_vsync_start = mode->crtc_vblank_start + sync_pos; - mode->crtc_vsync_end = mode->crtc_vsync_start + sync_width; + adjusted_mode->crtc_vsync_start = adjusted_mode->crtc_vblank_start + sync_pos; + adjusted_mode->crtc_vsync_end = adjusted_mode->crtc_vsync_start + sync_width; } static inline u32 panel_fitter_scaling(u32 source, u32 target) @@ -227,11 +226,11 @@ static inline u32 panel_fitter_scaling(u32 source, u32 target) static void i965_scale_aspect(struct intel_crtc_state *pipe_config, u32 *pfit_control) { - struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; - u32 scaled_width = adjusted_mode->hdisplay * + const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + u32 scaled_width = adjusted_mode->crtc_hdisplay * pipe_config->pipe_src_h; u32 scaled_height = pipe_config->pipe_src_w * - adjusted_mode->vdisplay; + adjusted_mode->crtc_vdisplay; /* 965+ is easy, it does everything in hw */ if (scaled_width > scaled_height) @@ -240,7 +239,7 @@ static void i965_scale_aspect(struct intel_crtc_state *pipe_config, else if (scaled_width < scaled_height) *pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER; - else if (adjusted_mode->hdisplay != pipe_config->pipe_src_w) + else if (adjusted_mode->crtc_hdisplay != pipe_config->pipe_src_w) *pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO; } @@ -249,10 +248,10 @@ static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, u32 *border) { struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; - u32 scaled_width = adjusted_mode->hdisplay * + u32 scaled_width = adjusted_mode->crtc_hdisplay * pipe_config->pipe_src_h; u32 scaled_height = pipe_config->pipe_src_w * - adjusted_mode->vdisplay; + adjusted_mode->crtc_vdisplay; u32 bits; /* @@ -266,9 +265,9 @@ static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, pipe_config->pipe_src_h); *border = LVDS_BORDER_ENABLE; - if (pipe_config->pipe_src_h != adjusted_mode->vdisplay) { + if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay) { bits = panel_fitter_scaling(pipe_config->pipe_src_h, - adjusted_mode->vdisplay); + adjusted_mode->crtc_vdisplay); *pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT | bits << PFIT_VERT_SCALE_SHIFT); @@ -282,9 +281,9 @@ static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, pipe_config->pipe_src_w); *border = LVDS_BORDER_ENABLE; - if (pipe_config->pipe_src_w != adjusted_mode->hdisplay) { + if (pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) { bits = panel_fitter_scaling(pipe_config->pipe_src_w, - adjusted_mode->hdisplay); + adjusted_mode->crtc_hdisplay); *pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT | bits << PFIT_VERT_SCALE_SHIFT); @@ -307,13 +306,11 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, { struct drm_device *dev = intel_crtc->base.dev; u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0; - struct drm_display_mode *adjusted_mode; - - adjusted_mode = &pipe_config->base.adjusted_mode; + struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; /* Native modes don't need fitting */ - if (adjusted_mode->hdisplay == pipe_config->pipe_src_w && - adjusted_mode->vdisplay == pipe_config->pipe_src_h) + if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w && + adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h) goto out; switch (fitting_mode) { @@ -339,8 +336,8 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, * Full scaling, even if it changes the aspect ratio. * Fortunately this is all done for us in hw. */ - if (pipe_config->pipe_src_h != adjusted_mode->vdisplay || - pipe_config->pipe_src_w != adjusted_mode->hdisplay) { + if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay || + pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) { pfit_control |= PFIT_ENABLE; if (INTEL_INFO(dev)->gen >= 4) pfit_control |= PFIT_SCALING_AUTO; @@ -384,7 +381,7 @@ intel_panel_detect(struct drm_device *dev) /* Assume that the BIOS does not lie through the OpRegion... */ if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) { - return ioread32(dev_priv->opregion.lid_state) & 0x1 ? + return *dev_priv->opregion.lid_state & 0x1 ? connector_status_connected : connector_status_disconnected; } @@ -481,7 +478,7 @@ static u32 intel_panel_compute_brightness(struct intel_connector *connector, return val; } -static u32 bdw_get_backlight(struct intel_connector *connector) +static u32 lpt_get_backlight(struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -539,9 +536,19 @@ static u32 vlv_get_backlight(struct intel_connector *connector) static u32 bxt_get_backlight(struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; + struct intel_panel *panel = &connector->panel; struct drm_i915_private *dev_priv = dev->dev_private; - return I915_READ(BXT_BLC_PWM_DUTY1); + return I915_READ(BXT_BLC_PWM_DUTY(panel->backlight.controller)); +} + +static u32 pwm_get_backlight(struct intel_connector *connector) +{ + struct intel_panel *panel = &connector->panel; + int duty_ns; + + duty_ns = pwm_get_duty_cycle(panel->backlight.pwm); + return DIV_ROUND_UP(duty_ns * 100, CRC_PMIC_PWM_PERIOD_NS); } static u32 intel_panel_get_backlight(struct intel_connector *connector) @@ -554,7 +561,7 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector) mutex_lock(&dev_priv->backlight_lock); if (panel->backlight.enabled) { - val = dev_priv->display.get_backlight(connector); + val = panel->backlight.get(connector); val = intel_panel_compute_brightness(connector, val); } @@ -564,7 +571,7 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector) return val; } -static void bdw_set_backlight(struct intel_connector *connector, u32 level) +static void lpt_set_backlight(struct intel_connector *connector, u32 level) { struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -628,20 +635,28 @@ static void bxt_set_backlight(struct intel_connector *connector, u32 level) { struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_panel *panel = &connector->panel; - I915_WRITE(BXT_BLC_PWM_DUTY1, level); + I915_WRITE(BXT_BLC_PWM_DUTY(panel->backlight.controller), level); +} + +static void pwm_set_backlight(struct intel_connector *connector, u32 level) +{ + struct intel_panel *panel = &connector->panel; + int duty_ns = DIV_ROUND_UP(level * CRC_PMIC_PWM_PERIOD_NS, 100); + + pwm_config(panel->backlight.pwm, duty_ns, CRC_PMIC_PWM_PERIOD_NS); } static void intel_panel_actually_set_backlight(struct intel_connector *connector, u32 level) { - struct drm_device *dev = connector->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_panel *panel = &connector->panel; DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level); level = intel_panel_compute_brightness(connector, level); - dev_priv->display.set_backlight(connector, level); + panel->backlight.set(connector, level); } /* set backlight brightness to level in range [0..max], scaling wrt hw min */ @@ -709,6 +724,32 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector, mutex_unlock(&dev_priv->backlight_lock); } +static void lpt_disable_backlight(struct intel_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 tmp; + + intel_panel_actually_set_backlight(connector, 0); + + /* + * Although we don't support or enable CPU PWM with LPT/SPT based + * systems, it may have been enabled prior to loading the + * driver. Disable to avoid warnings on LCPLL disable. + * + * This needs rework if we need to add support for CPU PWM on PCH split + * platforms. + */ + tmp = I915_READ(BLC_PWM_CPU_CTL2); + if (tmp & BLM_PWM_ENABLE) { + DRM_DEBUG_KMS("cpu backlight was enabled, disabling\n"); + I915_WRITE(BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE); + } + + tmp = I915_READ(BLC_PWM_PCH_CTL1); + I915_WRITE(BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE); +} + static void pch_disable_backlight(struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; @@ -761,12 +802,30 @@ static void bxt_disable_backlight(struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 tmp; + struct intel_panel *panel = &connector->panel; + u32 tmp, val; intel_panel_actually_set_backlight(connector, 0); - tmp = I915_READ(BXT_BLC_PWM_CTL1); - I915_WRITE(BXT_BLC_PWM_CTL1, tmp & ~BXT_BLC_PWM_ENABLE); + tmp = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller)); + I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller), + tmp & ~BXT_BLC_PWM_ENABLE); + + if (panel->backlight.controller == 1) { + val = I915_READ(UTIL_PIN_CTL); + val &= ~UTIL_PIN_ENABLE; + I915_WRITE(UTIL_PIN_CTL, val); + } +} + +static void pwm_disable_backlight(struct intel_connector *connector) +{ + struct intel_panel *panel = &connector->panel; + + /* Disable the backlight */ + pwm_config(panel->backlight.pwm, 0, CRC_PMIC_PWM_PERIOD_NS); + usleep_range(2000, 3000); + pwm_disable(panel->backlight.pwm); } void intel_panel_disable_backlight(struct intel_connector *connector) @@ -779,7 +838,7 @@ void intel_panel_disable_backlight(struct intel_connector *connector) return; /* - * Do not disable backlight on the vgaswitcheroo path. When switching + * Do not disable backlight on the vga_switcheroo path. When switching * away from i915, the other client may depend on i915 to handle the * backlight. This will leave the backlight on unnecessarily when * another client is not activated. @@ -794,12 +853,12 @@ void intel_panel_disable_backlight(struct intel_connector *connector) if (panel->backlight.device) panel->backlight.device->props.power = FB_BLANK_POWERDOWN; panel->backlight.enabled = false; - dev_priv->display.disable_backlight(connector); + panel->backlight.disable(connector); mutex_unlock(&dev_priv->backlight_lock); } -static void bdw_enable_backlight(struct intel_connector *connector) +static void lpt_enable_backlight(struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -988,16 +1047,38 @@ static void bxt_enable_backlight(struct intel_connector *connector) struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_panel *panel = &connector->panel; - u32 pwm_ctl; + enum pipe pipe = intel_get_pipe_from_connector(connector); + u32 pwm_ctl, val; + + /* To use 2nd set of backlight registers, utility pin has to be + * enabled with PWM mode. + * The field should only be changed when the utility pin is disabled + */ + if (panel->backlight.controller == 1) { + val = I915_READ(UTIL_PIN_CTL); + if (val & UTIL_PIN_ENABLE) { + DRM_DEBUG_KMS("util pin already enabled\n"); + val &= ~UTIL_PIN_ENABLE; + I915_WRITE(UTIL_PIN_CTL, val); + } - pwm_ctl = I915_READ(BXT_BLC_PWM_CTL1); + val = 0; + if (panel->backlight.util_pin_active_low) + val |= UTIL_PIN_POLARITY; + I915_WRITE(UTIL_PIN_CTL, val | UTIL_PIN_PIPE(pipe) | + UTIL_PIN_MODE_PWM | UTIL_PIN_ENABLE); + } + + pwm_ctl = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller)); if (pwm_ctl & BXT_BLC_PWM_ENABLE) { DRM_DEBUG_KMS("backlight already enabled\n"); pwm_ctl &= ~BXT_BLC_PWM_ENABLE; - I915_WRITE(BXT_BLC_PWM_CTL1, pwm_ctl); + I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl); } - I915_WRITE(BXT_BLC_PWM_FREQ1, panel->backlight.max); + I915_WRITE(BXT_BLC_PWM_FREQ(panel->backlight.controller), + panel->backlight.max); intel_panel_actually_set_backlight(connector, panel->backlight.level); @@ -1005,9 +1086,18 @@ static void bxt_enable_backlight(struct intel_connector *connector) if (panel->backlight.active_low_pwm) pwm_ctl |= BXT_BLC_PWM_POLARITY; - I915_WRITE(BXT_BLC_PWM_CTL1, pwm_ctl); - POSTING_READ(BXT_BLC_PWM_CTL1); - I915_WRITE(BXT_BLC_PWM_CTL1, pwm_ctl | BXT_BLC_PWM_ENABLE); + I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller), pwm_ctl); + POSTING_READ(BXT_BLC_PWM_CTL(panel->backlight.controller)); + I915_WRITE(BXT_BLC_PWM_CTL(panel->backlight.controller), + pwm_ctl | BXT_BLC_PWM_ENABLE); +} + +static void pwm_enable_backlight(struct intel_connector *connector) +{ + struct intel_panel *panel = &connector->panel; + + pwm_enable(panel->backlight.pwm); + intel_panel_actually_set_backlight(connector, panel->backlight.level); } void intel_panel_enable_backlight(struct intel_connector *connector) @@ -1035,7 +1125,7 @@ void intel_panel_enable_backlight(struct intel_connector *connector) panel->backlight.device->props.max_brightness); } - dev_priv->display.enable_backlight(connector); + panel->backlight.enable(connector); panel->backlight.enabled = true; if (panel->backlight.device) panel->backlight.device->props.power = FB_BLANK_UNBLANK; @@ -1063,10 +1153,10 @@ static int intel_backlight_device_update_status(struct backlight_device *bd) * callback needs to take this into account. */ if (panel->backlight.enabled) { - if (panel->backlight_power) { + if (panel->backlight.power) { bool enable = bd->props.power == FB_BLANK_UNBLANK && bd->props.brightness != 0; - panel->backlight_power(connector, enable); + panel->backlight.power(connector, enable); } } else { bd->props.power = FB_BLANK_POWERDOWN; @@ -1174,10 +1264,166 @@ static void intel_backlight_device_unregister(struct intel_connector *connector) #endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ /* - * Note: The setup hooks can't assume pipe is set! + * BXT: PWM clock frequency = 19.2 MHz. + */ +static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + return KHz(19200) / pwm_freq_hz; +} + +/* + * SPT: This value represents the period of the PWM stream in clock periods + * multiplied by 16 (default increment) or 128 (alternate increment selected in + * SCHICKEN_1 bit 0). PWM clock is 24 MHz. + */ +static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 mul, clock; + + if (I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY) + mul = 128; + else + mul = 16; + + clock = MHz(24); + + return clock / (pwm_freq_hz * mul); +} + +/* + * LPT: This value represents the period of the PWM stream in clock periods + * multiplied by 128 (default increment) or 16 (alternate increment, selected in + * LPT SOUTH_CHICKEN2 register bit 5). + */ +static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 mul, clock; + + if (I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY) + mul = 16; + else + mul = 128; + + if (HAS_PCH_LPT_H(dev_priv)) + clock = MHz(135); /* LPT:H */ + else + clock = MHz(24); /* LPT:LP */ + + return clock / (pwm_freq_hz * mul); +} + +/* + * ILK/SNB/IVB: This value represents the period of the PWM stream in PCH + * display raw clocks multiplied by 128. + */ +static u32 pch_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_device *dev = connector->base.dev; + int clock = MHz(intel_pch_rawclk(dev)); + + return clock / (pwm_freq_hz * 128); +} + +/* + * Gen2: This field determines the number of time base events (display core + * clock frequency/32) in total for a complete cycle of modulated backlight + * control. * - * XXX: Query mode clock or hardware clock and program PWM modulation frequency - * appropriately when it's 0. Use VBT and/or sane defaults. + * Gen3: A time base event equals the display core clock ([DevPNV] HRAW clock) + * divided by 32. + */ +static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int clock; + + if (IS_PINEVIEW(dev)) + clock = MHz(intel_hrawclk(dev)); + else + clock = 1000 * dev_priv->cdclk_freq; + + return clock / (pwm_freq_hz * 32); +} + +/* + * Gen4: This value represents the period of the PWM stream in display core + * clocks ([DevCTG] HRAW clocks) multiplied by 128. + * + */ +static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int clock; + + if (IS_G4X(dev_priv)) + clock = MHz(intel_hrawclk(dev)); + else + clock = 1000 * dev_priv->cdclk_freq; + + return clock / (pwm_freq_hz * 128); +} + +/* + * VLV: This value represents the period of the PWM stream in display core + * clocks ([DevCTG] 200MHz HRAW clocks) multiplied by 128 or 25MHz S0IX clocks + * multiplied by 16. CHV uses a 19.2MHz S0IX clock. + */ +static u32 vlv_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int clock; + + if ((I915_READ(CBR1_VLV) & CBR_PWM_CLOCK_MUX_SELECT) == 0) { + if (IS_CHERRYVIEW(dev)) + return KHz(19200) / (pwm_freq_hz * 16); + else + return MHz(25) / (pwm_freq_hz * 16); + } else { + clock = intel_hrawclk(dev); + return MHz(clock) / (pwm_freq_hz * 128); + } +} + +static u32 get_backlight_max_vbt(struct intel_connector *connector) +{ + struct drm_device *dev = connector->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_panel *panel = &connector->panel; + u16 pwm_freq_hz = dev_priv->vbt.backlight.pwm_freq_hz; + u32 pwm; + + if (!panel->backlight.hz_to_pwm) { + DRM_DEBUG_KMS("backlight frequency conversion not supported\n"); + return 0; + } + + if (pwm_freq_hz) { + DRM_DEBUG_KMS("VBT defined backlight frequency %u Hz\n", + pwm_freq_hz); + } else { + pwm_freq_hz = 200; + DRM_DEBUG_KMS("default backlight frequency %u Hz\n", + pwm_freq_hz); + } + + pwm = panel->backlight.hz_to_pwm(connector, pwm_freq_hz); + if (!pwm) { + DRM_DEBUG_KMS("backlight frequency conversion failed\n"); + return 0; + } + + return pwm; +} + +/* + * Note: The setup hooks can't assume pipe is set! */ static u32 get_backlight_min_vbt(struct intel_connector *connector) { @@ -1205,7 +1451,7 @@ static u32 get_backlight_min_vbt(struct intel_connector *connector) return scale(min, 0, 255, 0, panel->backlight.max); } -static int bdw_setup_backlight(struct intel_connector *connector, enum pipe unused) +static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unused) { struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1217,12 +1463,16 @@ static int bdw_setup_backlight(struct intel_connector *connector, enum pipe unus pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2); panel->backlight.max = pch_ctl2 >> 16; + + if (!panel->backlight.max) + panel->backlight.max = get_backlight_max_vbt(connector); + if (!panel->backlight.max) return -ENODEV; panel->backlight.min = get_backlight_min_vbt(connector); - val = bdw_get_backlight(connector); + val = lpt_get_backlight(connector); panel->backlight.level = intel_panel_compute_brightness(connector, val); panel->backlight.enabled = (pch_ctl1 & BLM_PCH_PWM_ENABLE) && @@ -1243,6 +1493,10 @@ static int pch_setup_backlight(struct intel_connector *connector, enum pipe unus pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2); panel->backlight.max = pch_ctl2 >> 16; + + if (!panel->backlight.max) + panel->backlight.max = get_backlight_max_vbt(connector); + if (!panel->backlight.max) return -ENODEV; @@ -1274,12 +1528,18 @@ static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unu panel->backlight.active_low_pwm = ctl & BLM_POLARITY_PNV; panel->backlight.max = ctl >> 17; - if (panel->backlight.combination_mode) - panel->backlight.max *= 0xff; + + if (!panel->backlight.max) { + panel->backlight.max = get_backlight_max_vbt(connector); + panel->backlight.max >>= 1; + } if (!panel->backlight.max) return -ENODEV; + if (panel->backlight.combination_mode) + panel->backlight.max *= 0xff; + panel->backlight.min = get_backlight_min_vbt(connector); val = i9xx_get_backlight(connector); @@ -1303,12 +1563,16 @@ static int i965_setup_backlight(struct intel_connector *connector, enum pipe unu ctl = I915_READ(BLC_PWM_CTL); panel->backlight.max = ctl >> 16; - if (panel->backlight.combination_mode) - panel->backlight.max *= 0xff; + + if (!panel->backlight.max) + panel->backlight.max = get_backlight_max_vbt(connector); if (!panel->backlight.max) return -ENODEV; + if (panel->backlight.combination_mode) + panel->backlight.max *= 0xff; + panel->backlight.min = get_backlight_min_vbt(connector); val = i9xx_get_backlight(connector); @@ -1325,21 +1589,8 @@ static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe struct drm_device *dev = connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_panel *panel = &connector->panel; - enum pipe p; u32 ctl, ctl2, val; - for_each_pipe(dev_priv, p) { - u32 cur_val = I915_READ(VLV_BLC_PWM_CTL(p)); - - /* Skip if the modulation freq is already set */ - if (cur_val & ~BACKLIGHT_DUTY_CYCLE_MASK) - continue; - - cur_val &= BACKLIGHT_DUTY_CYCLE_MASK; - I915_WRITE(VLV_BLC_PWM_CTL(p), (0xf42 << 16) | - cur_val); - } - if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B)) return -ENODEV; @@ -1348,6 +1599,10 @@ static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe ctl = I915_READ(VLV_BLC_PWM_CTL(pipe)); panel->backlight.max = ctl >> 16; + + if (!panel->backlight.max) + panel->backlight.max = get_backlight_max_vbt(connector); + if (!panel->backlight.max) return -ENODEV; @@ -1370,10 +1625,32 @@ bxt_setup_backlight(struct intel_connector *connector, enum pipe unused) struct intel_panel *panel = &connector->panel; u32 pwm_ctl, val; - pwm_ctl = I915_READ(BXT_BLC_PWM_CTL1); + /* + * For BXT hard coding the Backlight controller to 0. + * TODO : Read the controller value from VBT and generalize + */ + panel->backlight.controller = 0; + + pwm_ctl = I915_READ(BXT_BLC_PWM_CTL(panel->backlight.controller)); + + /* Keeping the check if controller 1 is to be programmed. + * This will come into affect once the VBT parsing + * is fixed for controller selection, and controller 1 is used + * for a prticular display configuration. + */ + if (panel->backlight.controller == 1) { + val = I915_READ(UTIL_PIN_CTL); + panel->backlight.util_pin_active_low = + val & UTIL_PIN_POLARITY; + } + panel->backlight.active_low_pwm = pwm_ctl & BXT_BLC_PWM_POLARITY; + panel->backlight.max = + I915_READ(BXT_BLC_PWM_FREQ(panel->backlight.controller)); + + if (!panel->backlight.max) + panel->backlight.max = get_backlight_max_vbt(connector); - panel->backlight.max = I915_READ(BXT_BLC_PWM_FREQ1); if (!panel->backlight.max) return -ENODEV; @@ -1386,6 +1663,40 @@ bxt_setup_backlight(struct intel_connector *connector, enum pipe unused) return 0; } +static int pwm_setup_backlight(struct intel_connector *connector, + enum pipe pipe) +{ + struct drm_device *dev = connector->base.dev; + struct intel_panel *panel = &connector->panel; + int retval; + + /* Get the PWM chip for backlight control */ + panel->backlight.pwm = pwm_get(dev->dev, "pwm_backlight"); + if (IS_ERR(panel->backlight.pwm)) { + DRM_ERROR("Failed to own the pwm chip\n"); + panel->backlight.pwm = NULL; + return -ENODEV; + } + + retval = pwm_config(panel->backlight.pwm, CRC_PMIC_PWM_PERIOD_NS, + CRC_PMIC_PWM_PERIOD_NS); + if (retval < 0) { + DRM_ERROR("Failed to configure the pwm chip\n"); + pwm_put(panel->backlight.pwm); + panel->backlight.pwm = NULL; + return retval; + } + + panel->backlight.min = 0; /* 0% */ + panel->backlight.max = 100; /* 100% */ + panel->backlight.level = DIV_ROUND_UP( + pwm_get_duty_cycle(panel->backlight.pwm) * 100, + CRC_PMIC_PWM_PERIOD_NS); + panel->backlight.enabled = panel->backlight.level != 0; + + return 0; +} + int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) { struct drm_device *dev = connector->dev; @@ -1403,9 +1714,13 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe) } } + /* ensure intel_panel has been initialized first */ + if (WARN_ON(!panel->backlight.setup)) + return -ENODEV; + /* set level and max in panel struct */ mutex_lock(&dev_priv->backlight_lock); - ret = dev_priv->display.setup_backlight(intel_connector, pipe); + ret = panel->backlight.setup(intel_connector, pipe); mutex_unlock(&dev_priv->backlight_lock); if (ret) { @@ -1429,50 +1744,75 @@ void intel_panel_destroy_backlight(struct drm_connector *connector) struct intel_connector *intel_connector = to_intel_connector(connector); struct intel_panel *panel = &intel_connector->panel; + /* dispose of the pwm */ + if (panel->backlight.pwm) + pwm_put(panel->backlight.pwm); + panel->backlight.present = false; } /* Set up chip specific backlight functions */ -void intel_panel_init_backlight_funcs(struct drm_device *dev) +static void +intel_panel_init_backlight_funcs(struct intel_panel *panel) { + struct intel_connector *intel_connector = + container_of(panel, struct intel_connector, panel); + struct drm_device *dev = intel_connector->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; if (IS_BROXTON(dev)) { - dev_priv->display.setup_backlight = bxt_setup_backlight; - dev_priv->display.enable_backlight = bxt_enable_backlight; - dev_priv->display.disable_backlight = bxt_disable_backlight; - dev_priv->display.set_backlight = bxt_set_backlight; - dev_priv->display.get_backlight = bxt_get_backlight; - } else if (IS_BROADWELL(dev) || IS_SKYLAKE(dev)) { - dev_priv->display.setup_backlight = bdw_setup_backlight; - dev_priv->display.enable_backlight = bdw_enable_backlight; - dev_priv->display.disable_backlight = pch_disable_backlight; - dev_priv->display.set_backlight = bdw_set_backlight; - dev_priv->display.get_backlight = bdw_get_backlight; + panel->backlight.setup = bxt_setup_backlight; + panel->backlight.enable = bxt_enable_backlight; + panel->backlight.disable = bxt_disable_backlight; + panel->backlight.set = bxt_set_backlight; + panel->backlight.get = bxt_get_backlight; + panel->backlight.hz_to_pwm = bxt_hz_to_pwm; + } else if (HAS_PCH_LPT(dev) || HAS_PCH_SPT(dev)) { + panel->backlight.setup = lpt_setup_backlight; + panel->backlight.enable = lpt_enable_backlight; + panel->backlight.disable = lpt_disable_backlight; + panel->backlight.set = lpt_set_backlight; + panel->backlight.get = lpt_get_backlight; + if (HAS_PCH_LPT(dev)) + panel->backlight.hz_to_pwm = lpt_hz_to_pwm; + else + panel->backlight.hz_to_pwm = spt_hz_to_pwm; } else if (HAS_PCH_SPLIT(dev)) { - dev_priv->display.setup_backlight = pch_setup_backlight; - dev_priv->display.enable_backlight = pch_enable_backlight; - dev_priv->display.disable_backlight = pch_disable_backlight; - dev_priv->display.set_backlight = pch_set_backlight; - dev_priv->display.get_backlight = pch_get_backlight; + panel->backlight.setup = pch_setup_backlight; + panel->backlight.enable = pch_enable_backlight; + panel->backlight.disable = pch_disable_backlight; + panel->backlight.set = pch_set_backlight; + panel->backlight.get = pch_get_backlight; + panel->backlight.hz_to_pwm = pch_hz_to_pwm; } else if (IS_VALLEYVIEW(dev)) { - dev_priv->display.setup_backlight = vlv_setup_backlight; - dev_priv->display.enable_backlight = vlv_enable_backlight; - dev_priv->display.disable_backlight = vlv_disable_backlight; - dev_priv->display.set_backlight = vlv_set_backlight; - dev_priv->display.get_backlight = vlv_get_backlight; + if (dev_priv->vbt.has_mipi) { + panel->backlight.setup = pwm_setup_backlight; + panel->backlight.enable = pwm_enable_backlight; + panel->backlight.disable = pwm_disable_backlight; + panel->backlight.set = pwm_set_backlight; + panel->backlight.get = pwm_get_backlight; + } else { + panel->backlight.setup = vlv_setup_backlight; + panel->backlight.enable = vlv_enable_backlight; + panel->backlight.disable = vlv_disable_backlight; + panel->backlight.set = vlv_set_backlight; + panel->backlight.get = vlv_get_backlight; + panel->backlight.hz_to_pwm = vlv_hz_to_pwm; + } } else if (IS_GEN4(dev)) { - dev_priv->display.setup_backlight = i965_setup_backlight; - dev_priv->display.enable_backlight = i965_enable_backlight; - dev_priv->display.disable_backlight = i965_disable_backlight; - dev_priv->display.set_backlight = i9xx_set_backlight; - dev_priv->display.get_backlight = i9xx_get_backlight; + panel->backlight.setup = i965_setup_backlight; + panel->backlight.enable = i965_enable_backlight; + panel->backlight.disable = i965_disable_backlight; + panel->backlight.set = i9xx_set_backlight; + panel->backlight.get = i9xx_get_backlight; + panel->backlight.hz_to_pwm = i965_hz_to_pwm; } else { - dev_priv->display.setup_backlight = i9xx_setup_backlight; - dev_priv->display.enable_backlight = i9xx_enable_backlight; - dev_priv->display.disable_backlight = i9xx_disable_backlight; - dev_priv->display.set_backlight = i9xx_set_backlight; - dev_priv->display.get_backlight = i9xx_get_backlight; + panel->backlight.setup = i9xx_setup_backlight; + panel->backlight.enable = i9xx_enable_backlight; + panel->backlight.disable = i9xx_disable_backlight; + panel->backlight.set = i9xx_set_backlight; + panel->backlight.get = i9xx_get_backlight; + panel->backlight.hz_to_pwm = i9xx_hz_to_pwm; } } @@ -1480,6 +1820,8 @@ int intel_panel_init(struct intel_panel *panel, struct drm_display_mode *fixed_mode, struct drm_display_mode *downclock_mode) { + intel_panel_init_backlight_funcs(panel); + panel->fixed_mode = fixed_mode; panel->downclock_mode = downclock_mode; diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index eadc15cddbeb..ee05ce8bf79a 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -52,69 +52,28 @@ #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) -static void gen9_init_clock_gating(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - /* WaEnableLbsSlaRetryTimerDecrement:skl */ - I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) | - GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); -} - -static void skl_init_clock_gating(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - gen9_init_clock_gating(dev); - - if (INTEL_REVID(dev) <= SKL_REVID_B0) { - /* - * WaDisableSDEUnitClockGating:skl - * WaSetGAPSunitClckGateDisable:skl - */ - I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | - GEN8_GAPSUNIT_CLOCK_GATE_DISABLE | - GEN8_SDEUNIT_CLOCK_GATE_DISABLE); - - /* WaDisableVFUnitClockGating:skl */ - I915_WRITE(GEN6_UCGCTL2, I915_READ(GEN6_UCGCTL2) | - GEN6_VFUNIT_CLOCK_GATE_DISABLE); - } - - if (INTEL_REVID(dev) <= SKL_REVID_D0) { - /* WaDisableHDCInvalidation:skl */ - I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | - BDW_DISABLE_HDC_INVALIDATION); - - /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */ - I915_WRITE(FF_SLICE_CS_CHICKEN2, - _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE)); - } - - if (INTEL_REVID(dev) <= SKL_REVID_E0) - /* WaDisableLSQCROPERFforOCL:skl */ - I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | - GEN8_LQSC_RO_PERF_DIS); -} - static void bxt_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - gen9_init_clock_gating(dev); + /* WaDisableSDEUnitClockGating:bxt */ + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* * FIXME: - * GEN8_SDEUNIT_CLOCK_GATE_DISABLE applies on A0 only. * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only. */ - /* WaDisableSDEUnitClockGating:bxt */ I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | - GEN8_SDEUNIT_CLOCK_GATE_DISABLE | GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ); - /* FIXME: apply on A0 only */ - I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF); + /* + * Wa: Backlight PWM may stop in the asserted state, causing backlight + * to stay fully on. + */ + if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) + I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | + PWM1_GATING_DIS | PWM2_GATING_DIS); } static void i915_pineview_get_mem_freq(struct drm_device *dev) @@ -334,22 +293,26 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) if (IS_VALLEYVIEW(dev)) { I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0); - if (IS_CHERRYVIEW(dev)) - chv_set_memory_pm5(dev_priv, enable); + POSTING_READ(FW_BLC_SELF_VLV); + dev_priv->wm.vlv.cxsr = enable; } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) { I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); + POSTING_READ(FW_BLC_SELF); } else if (IS_PINEVIEW(dev)) { val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN; val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0; I915_WRITE(DSPFW3, val); + POSTING_READ(DSPFW3); } else if (IS_I945G(dev) || IS_I945GM(dev)) { val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) : _MASKED_BIT_DISABLE(FW_BLC_SELF_EN); I915_WRITE(FW_BLC_SELF, val); + POSTING_READ(FW_BLC_SELF); } else if (IS_I915GM(dev)) { val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) : _MASKED_BIT_DISABLE(INSTPM_SELF_EN); I915_WRITE(INSTPM, val); + POSTING_READ(INSTPM); } else { return; } @@ -674,12 +637,9 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) crtc = single_enabled_crtc(dev); if (crtc) { - const struct drm_display_mode *adjusted_mode; + const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8; - int clock; - - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; - clock = adjusted_mode->crtc_clock; + int clock = adjusted_mode->crtc_clock; /* Display SR */ wm = intel_calculate_wm(clock, &pineview_display_wm, @@ -923,223 +883,480 @@ static void vlv_write_wm_values(struct intel_crtc *crtc, FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI)); } - POSTING_READ(DSPFW1); + /* zero (unused) WM1 watermarks */ + I915_WRITE(DSPFW4, 0); + I915_WRITE(DSPFW5, 0); + I915_WRITE(DSPFW6, 0); + I915_WRITE(DSPHOWM1, 0); - dev_priv->wm.vlv = *wm; + POSTING_READ(DSPFW1); } #undef FW_WM_VLV -static uint8_t vlv_compute_drain_latency(struct drm_crtc *crtc, - struct drm_plane *plane) +enum vlv_wm_level { + VLV_WM_LEVEL_PM2, + VLV_WM_LEVEL_PM5, + VLV_WM_LEVEL_DDR_DVFS, +}; + +/* latency must be in 0.1us units. */ +static unsigned int vlv_wm_method2(unsigned int pixel_rate, + unsigned int pipe_htotal, + unsigned int horiz_pixels, + unsigned int bytes_per_pixel, + unsigned int latency) { - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int entries, prec_mult, drain_latency, pixel_size; - int clock = intel_crtc->config->base.adjusted_mode.crtc_clock; - const int high_precision = IS_CHERRYVIEW(dev) ? 16 : 64; + unsigned int ret; - /* - * FIXME the plane might have an fb - * but be invisible (eg. due to clipping) - */ - if (!intel_crtc->active || !plane->state->fb) - return 0; + ret = (latency * pixel_rate) / (pipe_htotal * 10000); + ret = (ret + 1) * horiz_pixels * bytes_per_pixel; + ret = DIV_ROUND_UP(ret, 64); - if (WARN(clock == 0, "Pixel clock is zero!\n")) - return 0; + return ret; +} - pixel_size = drm_format_plane_cpp(plane->state->fb->pixel_format, 0); +static void vlv_setup_wm_latency(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; - if (WARN(pixel_size == 0, "Pixel size is zero!\n")) - return 0; + /* all latencies in usec */ + dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; - entries = DIV_ROUND_UP(clock, 1000) * pixel_size; + dev_priv->wm.max_level = VLV_WM_LEVEL_PM2; - prec_mult = high_precision; - drain_latency = 64 * prec_mult * 4 / entries; + if (IS_CHERRYVIEW(dev_priv)) { + dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; + dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; - if (drain_latency > DRAIN_LATENCY_MASK) { - prec_mult /= 2; - drain_latency = 64 * prec_mult * 4 / entries; + dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS; } - - if (drain_latency > DRAIN_LATENCY_MASK) - drain_latency = DRAIN_LATENCY_MASK; - - return drain_latency | (prec_mult == high_precision ? - DDL_PRECISION_HIGH : DDL_PRECISION_LOW); } -static int vlv_compute_wm(struct intel_crtc *crtc, - struct intel_plane *plane, - int fifo_size) +static uint16_t vlv_compute_wm_level(struct intel_plane *plane, + struct intel_crtc *crtc, + const struct intel_plane_state *state, + int level) { - int clock, entries, pixel_size; + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + int clock, htotal, pixel_size, width, wm; - /* - * FIXME the plane might have an fb - * but be invisible (eg. due to clipping) - */ - if (!crtc->active || !plane->base.state->fb) + if (dev_priv->wm.pri_latency[level] == 0) + return USHRT_MAX; + + if (!state->visible) return 0; - pixel_size = drm_format_plane_cpp(plane->base.state->fb->pixel_format, 0); + pixel_size = drm_format_plane_cpp(state->base.fb->pixel_format, 0); clock = crtc->config->base.adjusted_mode.crtc_clock; + htotal = crtc->config->base.adjusted_mode.crtc_htotal; + width = crtc->config->pipe_src_w; + if (WARN_ON(htotal == 0)) + htotal = 1; - entries = DIV_ROUND_UP(clock, 1000) * pixel_size; + if (plane->base.type == DRM_PLANE_TYPE_CURSOR) { + /* + * FIXME the formula gives values that are + * too big for the cursor FIFO, and hence we + * would never be able to use cursors. For + * now just hardcode the watermark. + */ + wm = 63; + } else { + wm = vlv_wm_method2(clock, htotal, width, pixel_size, + dev_priv->wm.pri_latency[level] * 10); + } - /* - * Set up the watermark such that we don't start issuing memory - * requests until we are within PND's max deadline value (256us). - * Idea being to be idle as long as possible while still taking - * advatange of PND's deadline scheduling. The limit of 8 - * cachelines (used when the FIFO will anyway drain in less time - * than 256us) should match what we would be done if trickle - * feed were enabled. - */ - return fifo_size - clamp(DIV_ROUND_UP(256 * entries, 64), 0, fifo_size - 8); + return min_t(int, wm, USHRT_MAX); } -static bool vlv_compute_sr_wm(struct drm_device *dev, - struct vlv_wm_values *wm) +static void vlv_compute_fifo(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; - enum pipe pipe = INVALID_PIPE; - int num_planes = 0; - int fifo_size = 0; + struct drm_device *dev = crtc->base.dev; + struct vlv_wm_state *wm_state = &crtc->wm_state; struct intel_plane *plane; + unsigned int total_rate = 0; + const int fifo_size = 512 - 1; + int fifo_extra, fifo_left = fifo_size; - wm->sr.cursor = wm->sr.plane = 0; + for_each_intel_plane_on_crtc(dev, crtc, plane) { + struct intel_plane_state *state = + to_intel_plane_state(plane->base.state); - crtc = single_enabled_crtc(dev); - /* maxfifo not supported on pipe C */ - if (crtc && to_intel_crtc(crtc)->pipe != PIPE_C) { - pipe = to_intel_crtc(crtc)->pipe; - num_planes = !!wm->pipe[pipe].primary + - !!wm->pipe[pipe].sprite[0] + - !!wm->pipe[pipe].sprite[1]; - fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1; + if (plane->base.type == DRM_PLANE_TYPE_CURSOR) + continue; + + if (state->visible) { + wm_state->num_active_planes++; + total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0); + } } - if (fifo_size == 0 || num_planes > 1) - return false; + for_each_intel_plane_on_crtc(dev, crtc, plane) { + struct intel_plane_state *state = + to_intel_plane_state(plane->base.state); + unsigned int rate; + + if (plane->base.type == DRM_PLANE_TYPE_CURSOR) { + plane->wm.fifo_size = 63; + continue; + } + + if (!state->visible) { + plane->wm.fifo_size = 0; + continue; + } + + rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0); + plane->wm.fifo_size = fifo_size * rate / total_rate; + fifo_left -= plane->wm.fifo_size; + } - wm->sr.cursor = vlv_compute_wm(to_intel_crtc(crtc), - to_intel_plane(crtc->cursor), 0x3f); + fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1); + + /* spread the remainder evenly */ + for_each_intel_plane_on_crtc(dev, crtc, plane) { + int plane_extra; + + if (fifo_left == 0) + break; - list_for_each_entry(plane, &dev->mode_config.plane_list, base.head) { if (plane->base.type == DRM_PLANE_TYPE_CURSOR) continue; - if (plane->pipe != pipe) + /* give it all to the first plane if none are active */ + if (plane->wm.fifo_size == 0 && + wm_state->num_active_planes) continue; - wm->sr.plane = vlv_compute_wm(to_intel_crtc(crtc), - plane, fifo_size); - if (wm->sr.plane != 0) + plane_extra = min(fifo_extra, fifo_left); + plane->wm.fifo_size += plane_extra; + fifo_left -= plane_extra; + } + + WARN_ON(fifo_left != 0); +} + +static void vlv_invert_wms(struct intel_crtc *crtc) +{ + struct vlv_wm_state *wm_state = &crtc->wm_state; + int level; + + for (level = 0; level < wm_state->num_levels; level++) { + struct drm_device *dev = crtc->base.dev; + const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1; + struct intel_plane *plane; + + wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane; + wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor; + + for_each_intel_plane_on_crtc(dev, crtc, plane) { + switch (plane->base.type) { + int sprite; + case DRM_PLANE_TYPE_CURSOR: + wm_state->wm[level].cursor = plane->wm.fifo_size - + wm_state->wm[level].cursor; + break; + case DRM_PLANE_TYPE_PRIMARY: + wm_state->wm[level].primary = plane->wm.fifo_size - + wm_state->wm[level].primary; + break; + case DRM_PLANE_TYPE_OVERLAY: + sprite = plane->plane; + wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size - + wm_state->wm[level].sprite[sprite]; + break; + } + } + } +} + +static void vlv_compute_wm(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct vlv_wm_state *wm_state = &crtc->wm_state; + struct intel_plane *plane; + int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1; + int level; + + memset(wm_state, 0, sizeof(*wm_state)); + + wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed; + wm_state->num_levels = to_i915(dev)->wm.max_level + 1; + + wm_state->num_active_planes = 0; + + vlv_compute_fifo(crtc); + + if (wm_state->num_active_planes != 1) + wm_state->cxsr = false; + + if (wm_state->cxsr) { + for (level = 0; level < wm_state->num_levels; level++) { + wm_state->sr[level].plane = sr_fifo_size; + wm_state->sr[level].cursor = 63; + } + } + + for_each_intel_plane_on_crtc(dev, crtc, plane) { + struct intel_plane_state *state = + to_intel_plane_state(plane->base.state); + + if (!state->visible) + continue; + + /* normal watermarks */ + for (level = 0; level < wm_state->num_levels; level++) { + int wm = vlv_compute_wm_level(plane, crtc, state, level); + int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511; + + /* hack */ + if (WARN_ON(level == 0 && wm > max_wm)) + wm = max_wm; + + if (wm > plane->wm.fifo_size) + break; + + switch (plane->base.type) { + int sprite; + case DRM_PLANE_TYPE_CURSOR: + wm_state->wm[level].cursor = wm; + break; + case DRM_PLANE_TYPE_PRIMARY: + wm_state->wm[level].primary = wm; + break; + case DRM_PLANE_TYPE_OVERLAY: + sprite = plane->plane; + wm_state->wm[level].sprite[sprite] = wm; + break; + } + } + + wm_state->num_levels = level; + + if (!wm_state->cxsr) + continue; + + /* maxfifo watermarks */ + switch (plane->base.type) { + int sprite, level; + case DRM_PLANE_TYPE_CURSOR: + for (level = 0; level < wm_state->num_levels; level++) + wm_state->sr[level].cursor = + wm_state->wm[level].cursor; break; + case DRM_PLANE_TYPE_PRIMARY: + for (level = 0; level < wm_state->num_levels; level++) + wm_state->sr[level].plane = + min(wm_state->sr[level].plane, + wm_state->wm[level].primary); + break; + case DRM_PLANE_TYPE_OVERLAY: + sprite = plane->plane; + for (level = 0; level < wm_state->num_levels; level++) + wm_state->sr[level].plane = + min(wm_state->sr[level].plane, + wm_state->wm[level].sprite[sprite]); + break; + } } - return true; + /* clear any (partially) filled invalid levels */ + for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) { + memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level])); + memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level])); + } + + vlv_invert_wms(crtc); } -static void valleyview_update_wm(struct drm_crtc *crtc) +#define VLV_FIFO(plane, value) \ + (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV) + +static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; - bool cxsr_enabled; - struct vlv_wm_values wm = dev_priv->wm.vlv; + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_plane *plane; + int sprite0_start = 0, sprite1_start = 0, fifo_size = 0; - wm.ddl[pipe].primary = vlv_compute_drain_latency(crtc, crtc->primary); - wm.pipe[pipe].primary = vlv_compute_wm(intel_crtc, - to_intel_plane(crtc->primary), - vlv_get_fifo_size(dev, pipe, 0)); + for_each_intel_plane_on_crtc(dev, crtc, plane) { + if (plane->base.type == DRM_PLANE_TYPE_CURSOR) { + WARN_ON(plane->wm.fifo_size != 63); + continue; + } - wm.ddl[pipe].cursor = vlv_compute_drain_latency(crtc, crtc->cursor); - wm.pipe[pipe].cursor = vlv_compute_wm(intel_crtc, - to_intel_plane(crtc->cursor), - 0x3f); + if (plane->base.type == DRM_PLANE_TYPE_PRIMARY) + sprite0_start = plane->wm.fifo_size; + else if (plane->plane == 0) + sprite1_start = sprite0_start + plane->wm.fifo_size; + else + fifo_size = sprite1_start + plane->wm.fifo_size; + } - cxsr_enabled = vlv_compute_sr_wm(dev, &wm); + WARN_ON(fifo_size != 512 - 1); - if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0) - return; + DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n", + pipe_name(crtc->pipe), sprite0_start, + sprite1_start, fifo_size); - DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, " - "SR: plane=%d, cursor=%d\n", pipe_name(pipe), - wm.pipe[pipe].primary, wm.pipe[pipe].cursor, - wm.sr.plane, wm.sr.cursor); + switch (crtc->pipe) { + uint32_t dsparb, dsparb2, dsparb3; + case PIPE_A: + dsparb = I915_READ(DSPARB); + dsparb2 = I915_READ(DSPARB2); - /* - * FIXME DDR DVFS introduces massive memory latencies which - * are not known to system agent so any deadline specified - * by the display may not be respected. To support DDR DVFS - * the watermark code needs to be rewritten to essentially - * bypass deadline mechanism and rely solely on the - * watermarks. For now disable DDR DVFS. - */ - if (IS_CHERRYVIEW(dev_priv)) - chv_set_memory_dvfs(dev_priv, false); + dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) | + VLV_FIFO(SPRITEB, 0xff)); + dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) | + VLV_FIFO(SPRITEB, sprite1_start)); - if (!cxsr_enabled) - intel_set_memory_cxsr(dev_priv, false); + dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) | + VLV_FIFO(SPRITEB_HI, 0x1)); + dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) | + VLV_FIFO(SPRITEB_HI, sprite1_start >> 8)); - vlv_write_wm_values(intel_crtc, &wm); + I915_WRITE(DSPARB, dsparb); + I915_WRITE(DSPARB2, dsparb2); + break; + case PIPE_B: + dsparb = I915_READ(DSPARB); + dsparb2 = I915_READ(DSPARB2); - if (cxsr_enabled) - intel_set_memory_cxsr(dev_priv, true); + dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) | + VLV_FIFO(SPRITED, 0xff)); + dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) | + VLV_FIFO(SPRITED, sprite1_start)); + + dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) | + VLV_FIFO(SPRITED_HI, 0xff)); + dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) | + VLV_FIFO(SPRITED_HI, sprite1_start >> 8)); + + I915_WRITE(DSPARB, dsparb); + I915_WRITE(DSPARB2, dsparb2); + break; + case PIPE_C: + dsparb3 = I915_READ(DSPARB3); + dsparb2 = I915_READ(DSPARB2); + + dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) | + VLV_FIFO(SPRITEF, 0xff)); + dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) | + VLV_FIFO(SPRITEF, sprite1_start)); + + dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) | + VLV_FIFO(SPRITEF_HI, 0xff)); + dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) | + VLV_FIFO(SPRITEF_HI, sprite1_start >> 8)); + + I915_WRITE(DSPARB3, dsparb3); + I915_WRITE(DSPARB2, dsparb2); + break; + default: + break; + } } -static void valleyview_update_sprite_wm(struct drm_plane *plane, - struct drm_crtc *crtc, - uint32_t sprite_width, - uint32_t sprite_height, - int pixel_size, - bool enabled, bool scaled) +#undef VLV_FIFO + +static void vlv_merge_wm(struct drm_device *dev, + struct vlv_wm_values *wm) +{ + struct intel_crtc *crtc; + int num_active_crtcs = 0; + + wm->level = to_i915(dev)->wm.max_level; + wm->cxsr = true; + + for_each_intel_crtc(dev, crtc) { + const struct vlv_wm_state *wm_state = &crtc->wm_state; + + if (!crtc->active) + continue; + + if (!wm_state->cxsr) + wm->cxsr = false; + + num_active_crtcs++; + wm->level = min_t(int, wm->level, wm_state->num_levels - 1); + } + + if (num_active_crtcs != 1) + wm->cxsr = false; + + if (num_active_crtcs > 1) + wm->level = VLV_WM_LEVEL_PM2; + + for_each_intel_crtc(dev, crtc) { + struct vlv_wm_state *wm_state = &crtc->wm_state; + enum pipe pipe = crtc->pipe; + + if (!crtc->active) + continue; + + wm->pipe[pipe] = wm_state->wm[wm->level]; + if (wm->cxsr) + wm->sr = wm_state->sr[wm->level]; + + wm->ddl[pipe].primary = DDL_PRECISION_HIGH | 2; + wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH | 2; + wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH | 2; + wm->ddl[pipe].cursor = DDL_PRECISION_HIGH | 2; + } +} + +static void vlv_update_wm(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; - int sprite = to_intel_plane(plane)->plane; - bool cxsr_enabled; - struct vlv_wm_values wm = dev_priv->wm.vlv; + struct vlv_wm_values wm = {}; - if (enabled) { - wm.ddl[pipe].sprite[sprite] = - vlv_compute_drain_latency(crtc, plane); + vlv_compute_wm(intel_crtc); + vlv_merge_wm(dev, &wm); - wm.pipe[pipe].sprite[sprite] = - vlv_compute_wm(intel_crtc, - to_intel_plane(plane), - vlv_get_fifo_size(dev, pipe, sprite+1)); - } else { - wm.ddl[pipe].sprite[sprite] = 0; - wm.pipe[pipe].sprite[sprite] = 0; + if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) { + /* FIXME should be part of crtc atomic commit */ + vlv_pipe_set_fifo_size(intel_crtc); + return; } - cxsr_enabled = vlv_compute_sr_wm(dev, &wm); - - if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0) - return; + if (wm.level < VLV_WM_LEVEL_DDR_DVFS && + dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS) + chv_set_memory_dvfs(dev_priv, false); - DRM_DEBUG_KMS("Setting FIFO watermarks - %c: sprite %c=%d, " - "SR: plane=%d, cursor=%d\n", pipe_name(pipe), - sprite_name(pipe, sprite), - wm.pipe[pipe].sprite[sprite], - wm.sr.plane, wm.sr.cursor); + if (wm.level < VLV_WM_LEVEL_PM5 && + dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5) + chv_set_memory_pm5(dev_priv, false); - if (!cxsr_enabled) + if (!wm.cxsr && dev_priv->wm.vlv.cxsr) intel_set_memory_cxsr(dev_priv, false); + /* FIXME should be part of crtc atomic commit */ + vlv_pipe_set_fifo_size(intel_crtc); + vlv_write_wm_values(intel_crtc, &wm); - if (cxsr_enabled) + DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, " + "sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n", + pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor, + wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1], + wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr); + + if (wm.cxsr && !dev_priv->wm.vlv.cxsr) intel_set_memory_cxsr(dev_priv, true); + + if (wm.level >= VLV_WM_LEVEL_PM5 && + dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5) + chv_set_memory_pm5(dev_priv, true); + + if (wm.level >= VLV_WM_LEVEL_DDR_DVFS && + dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS) + chv_set_memory_dvfs(dev_priv, true); + + dev_priv->wm.vlv = wm; } #define single_plane_enabled(mask) is_power_of_2(mask) @@ -1216,8 +1433,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) if (crtc) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 12000; - const struct drm_display_mode *adjusted_mode = - &to_intel_crtc(crtc)->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; @@ -1364,8 +1580,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) if (HAS_FW_BLC(dev) && enabled) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 6000; - const struct drm_display_mode *adjusted_mode = - &to_intel_crtc(enabled)->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode; int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w; @@ -1434,23 +1649,22 @@ static void i845_update_wm(struct drm_crtc *unused_crtc) I915_WRITE(FW_BLC, fwater_lo); } -static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev, - struct drm_crtc *crtc) +uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pixel_rate; - pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock; + pixel_rate = pipe_config->base.adjusted_mode.crtc_clock; /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to * adjust the pixel_rate here. */ - if (intel_crtc->config->pch_pfit.enabled) { + if (pipe_config->pch_pfit.enabled) { uint64_t pipe_w, pipe_h, pfit_w, pfit_h; - uint32_t pfit_size = intel_crtc->config->pch_pfit.size; + uint32_t pfit_size = pipe_config->pch_pfit.size; + + pipe_w = pipe_config->pipe_src_w; + pipe_h = pipe_config->pipe_src_h; - pipe_w = intel_crtc->config->pipe_src_w; - pipe_h = intel_crtc->config->pipe_src_h; pfit_w = (pfit_size >> 16) & 0xFFFF; pfit_h = pfit_size & 0xFFFF; if (pipe_w < pfit_w) @@ -1502,23 +1716,6 @@ static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels, return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2; } -struct skl_pipe_wm_parameters { - bool active; - uint32_t pipe_htotal; - uint32_t pixel_rate; /* in KHz */ - struct intel_plane_wm_parameters plane[I915_MAX_PLANES]; - struct intel_plane_wm_parameters cursor; -}; - -struct ilk_pipe_wm_parameters { - bool active; - uint32_t pipe_htotal; - uint32_t pixel_rate; - struct intel_plane_wm_parameters pri; - struct intel_plane_wm_parameters spr; - struct intel_plane_wm_parameters cur; -}; - struct ilk_wm_maximums { uint16_t pri; uint16_t spr; @@ -1526,37 +1723,30 @@ struct ilk_wm_maximums { uint16_t fbc; }; -/* used in computing the new watermarks state */ -struct intel_wm_config { - unsigned int num_pipes_active; - bool sprites_enabled; - bool sprites_scaled; -}; - /* * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params, +static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, uint32_t mem_value, bool is_lp) { + int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0; uint32_t method1, method2; - if (!params->active || !params->pri.enabled) + if (!cstate->base.active || !pstate->visible) return 0; - method1 = ilk_wm_method1(params->pixel_rate, - params->pri.bytes_per_pixel, - mem_value); + method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value); if (!is_lp) return method1; - method2 = ilk_wm_method2(params->pixel_rate, - params->pipe_htotal, - params->pri.horiz_pixels, - params->pri.bytes_per_pixel, + method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate), + cstate->base.adjusted_mode.crtc_htotal, + drm_rect_width(&pstate->dst), + bpp, mem_value); return min(method1, method2); @@ -1566,21 +1756,21 @@ static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params, * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params, +static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, uint32_t mem_value) { + int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0; uint32_t method1, method2; - if (!params->active || !params->spr.enabled) + if (!cstate->base.active || !pstate->visible) return 0; - method1 = ilk_wm_method1(params->pixel_rate, - params->spr.bytes_per_pixel, - mem_value); - method2 = ilk_wm_method2(params->pixel_rate, - params->pipe_htotal, - params->spr.horiz_pixels, - params->spr.bytes_per_pixel, + method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value); + method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate), + cstate->base.adjusted_mode.crtc_htotal, + drm_rect_width(&pstate->dst), + bpp, mem_value); return min(method1, method2); } @@ -1589,29 +1779,33 @@ static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params, * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ -static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params, +static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, uint32_t mem_value) { - if (!params->active || !params->cur.enabled) + int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0; + + if (!cstate->base.active || !pstate->visible) return 0; - return ilk_wm_method2(params->pixel_rate, - params->pipe_htotal, - params->cur.horiz_pixels, - params->cur.bytes_per_pixel, + return ilk_wm_method2(ilk_pipe_pixel_rate(cstate), + cstate->base.adjusted_mode.crtc_htotal, + drm_rect_width(&pstate->dst), + bpp, mem_value); } /* Only for WM_LP. */ -static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params, +static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate, + const struct intel_plane_state *pstate, uint32_t pri_val) { - if (!params->active || !params->pri.enabled) + int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0; + + if (!cstate->base.active || !pstate->visible) return 0; - return ilk_wm_fbc(pri_val, - params->pri.horiz_pixels, - params->pri.bytes_per_pixel); + return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), bpp); } static unsigned int ilk_display_fifo_size(const struct drm_device *dev) @@ -1776,8 +1970,12 @@ static bool ilk_validate_wm_level(int level, } static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, + const struct intel_crtc *intel_crtc, int level, - const struct ilk_pipe_wm_parameters *p, + struct intel_crtc_state *cstate, + struct intel_plane_state *pristate, + struct intel_plane_state *sprstate, + struct intel_plane_state *curstate, struct intel_wm_level *result) { uint16_t pri_latency = dev_priv->wm.pri_latency[level]; @@ -1791,10 +1989,11 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, cur_latency *= 5; } - result->pri_val = ilk_compute_pri_wm(p, pri_latency, level); - result->spr_val = ilk_compute_spr_wm(p, spr_latency); - result->cur_val = ilk_compute_cur_wm(p, cur_latency); - result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val); + result->pri_val = ilk_compute_pri_wm(cstate, pristate, + pri_latency, level); + result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency); + result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency); + result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val); result->enable = true; } @@ -1803,7 +2002,7 @@ hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; u32 linetime, ips_linetime; if (!intel_crtc->active) @@ -1812,10 +2011,10 @@ hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc) /* The WM are computed with base on how long it takes to fill a single * row at the given clock rate, multiplied by 8. * */ - linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8, - mode->crtc_clock); - ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8, - dev_priv->display.get_display_clock_speed(dev_priv->dev)); + linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, + adjusted_mode->crtc_clock); + ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, + dev_priv->cdclk_freq); return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) | PIPE_WM_LINETIME_TIME(linetime); @@ -2053,112 +2252,84 @@ static void skl_setup_wm_latency(struct drm_device *dev) intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); } -static void ilk_compute_wm_parameters(struct drm_crtc *crtc, - struct ilk_pipe_wm_parameters *p) -{ - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; - struct drm_plane *plane; - - if (!intel_crtc->active) - return; - - p->active = true; - p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal; - p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); - - if (crtc->primary->state->fb) - p->pri.bytes_per_pixel = - crtc->primary->state->fb->bits_per_pixel / 8; - else - p->pri.bytes_per_pixel = 4; - - p->cur.bytes_per_pixel = 4; - /* - * TODO: for now, assume primary and cursor planes are always enabled. - * Setting them to false makes the screen flicker. - */ - p->pri.enabled = true; - p->cur.enabled = true; - - p->pri.horiz_pixels = intel_crtc->config->pipe_src_w; - p->cur.horiz_pixels = intel_crtc->base.cursor->state->crtc_w; - - drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { - struct intel_plane *intel_plane = to_intel_plane(plane); - - if (intel_plane->pipe == pipe) { - p->spr = intel_plane->wm; - break; - } - } -} - -static void ilk_compute_wm_config(struct drm_device *dev, - struct intel_wm_config *config) -{ - struct intel_crtc *intel_crtc; - - /* Compute the currently _active_ config */ - for_each_intel_crtc(dev, intel_crtc) { - const struct intel_pipe_wm *wm = &intel_crtc->wm.active; - - if (!wm->pipe_enabled) - continue; - - config->sprites_enabled |= wm->sprites_enabled; - config->sprites_scaled |= wm->sprites_scaled; - config->num_pipes_active++; - } -} - /* Compute new watermarks for the pipe */ -static bool intel_compute_pipe_wm(struct drm_crtc *crtc, - const struct ilk_pipe_wm_parameters *params, - struct intel_pipe_wm *pipe_wm) +static int ilk_compute_pipe_wm(struct intel_crtc *intel_crtc, + struct drm_atomic_state *state) { - struct drm_device *dev = crtc->dev; + struct intel_pipe_wm *pipe_wm; + struct drm_device *dev = intel_crtc->base.dev; const struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc_state *cstate = NULL; + struct intel_plane *intel_plane; + struct drm_plane_state *ps; + struct intel_plane_state *pristate = NULL; + struct intel_plane_state *sprstate = NULL; + struct intel_plane_state *curstate = NULL; int level, max_level = ilk_wm_max_level(dev); /* LP0 watermark maximums depend on this pipe alone */ struct intel_wm_config config = { .num_pipes_active = 1, - .sprites_enabled = params->spr.enabled, - .sprites_scaled = params->spr.scaled, }; struct ilk_wm_maximums max; - pipe_wm->pipe_enabled = params->active; - pipe_wm->sprites_enabled = params->spr.enabled; - pipe_wm->sprites_scaled = params->spr.scaled; + cstate = intel_atomic_get_crtc_state(state, intel_crtc); + if (IS_ERR(cstate)) + return PTR_ERR(cstate); + + pipe_wm = &cstate->wm.optimal.ilk; + + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + ps = drm_atomic_get_plane_state(state, + &intel_plane->base); + if (IS_ERR(ps)) + return PTR_ERR(ps); + + if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY) + pristate = to_intel_plane_state(ps); + else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) + sprstate = to_intel_plane_state(ps); + else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR) + curstate = to_intel_plane_state(ps); + } + + config.sprites_enabled = sprstate->visible; + config.sprites_scaled = sprstate->visible && + (drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 || + drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16); + + pipe_wm->pipe_enabled = cstate->base.active; + pipe_wm->sprites_enabled = config.sprites_enabled; + pipe_wm->sprites_scaled = config.sprites_scaled; /* ILK/SNB: LP2+ watermarks only w/o sprites */ - if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled) + if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible) max_level = 1; /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ - if (params->spr.scaled) + if (config.sprites_scaled) max_level = 0; - ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]); + ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, + pristate, sprstate, curstate, &pipe_wm->wm[0]); if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc); + pipe_wm->linetime = hsw_compute_linetime_wm(dev, + &intel_crtc->base); /* LP0 watermarks always use 1/2 DDB partitioning */ ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); /* At least LP0 must be valid */ if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) - return false; + return -EINVAL; ilk_compute_wm_reg_maximums(dev, 1, &max); for (level = 1; level <= max_level; level++) { struct intel_wm_level wm = {}; - ilk_compute_wm_level(dev_priv, level, params, &wm); + ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, + pristate, sprstate, curstate, &wm); /* * Disable any watermark level that exceeds the @@ -2171,7 +2342,7 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, pipe_wm->wm[level] = wm; } - return true; + return 0; } /* @@ -2186,7 +2357,9 @@ static void ilk_merge_wm_level(struct drm_device *dev, ret_wm->enable = true; for_each_intel_crtc(dev, intel_crtc) { - const struct intel_pipe_wm *active = &intel_crtc->wm.active; + const struct intel_crtc_state *cstate = + to_intel_crtc_state(intel_crtc->base.state); + const struct intel_pipe_wm *active = &cstate->wm.optimal.ilk; const struct intel_wm_level *wm = &active->wm[level]; if (!active->pipe_enabled) @@ -2215,6 +2388,7 @@ static void ilk_wm_merge(struct drm_device *dev, const struct ilk_wm_maximums *max, struct intel_pipe_wm *merged) { + struct drm_i915_private *dev_priv = dev->dev_private; int level, max_level = ilk_wm_max_level(dev); int last_enabled_level = max_level; @@ -2255,7 +2429,8 @@ static void ilk_wm_merge(struct drm_device *dev, * What we should check here is whether FBC can be * enabled sometime later. */ - if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) { + if (IS_GEN5(dev) && !merged->fbc_wm_enabled && + intel_fbc_is_active(dev_priv)) { for (level = 2; level <= max_level; level++) { struct intel_wm_level *wm = &merged->wm[level]; @@ -2332,14 +2507,15 @@ static void ilk_compute_wm_results(struct drm_device *dev, /* LP0 register values */ for_each_intel_crtc(dev, intel_crtc) { + const struct intel_crtc_state *cstate = + to_intel_crtc_state(intel_crtc->base.state); enum pipe pipe = intel_crtc->pipe; - const struct intel_wm_level *r = - &intel_crtc->wm.active.wm[0]; + const struct intel_wm_level *r = &cstate->wm.optimal.ilk.wm[0]; if (WARN_ON(!r->enable)) continue; - results->wm_linetime[pipe] = intel_crtc->wm.active.linetime; + results->wm_linetime[pipe] = cstate->wm.optimal.ilk.linetime; results->wm_pipe[pipe] = (r->pri_val << WM0_PIPE_PLANE_SHIFT) | @@ -2561,18 +2737,40 @@ static bool ilk_disable_lp_wm(struct drm_device *dev) #define SKL_DDB_SIZE 896 /* in blocks */ #define BXT_DDB_SIZE 512 +/* + * Return the index of a plane in the SKL DDB and wm result arrays. Primary + * plane is always in slot 0, cursor is always in slot I915_MAX_PLANES-1, and + * other universal planes are in indices 1..n. Note that this may leave unused + * indices between the top "sprite" plane and the cursor. + */ +static int +skl_wm_plane_id(const struct intel_plane *plane) +{ + switch (plane->base.type) { + case DRM_PLANE_TYPE_PRIMARY: + return 0; + case DRM_PLANE_TYPE_CURSOR: + return PLANE_CURSOR; + case DRM_PLANE_TYPE_OVERLAY: + return plane->plane + 1; + default: + MISSING_CASE(plane->base.type); + return plane->plane; + } +} + static void skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, - struct drm_crtc *for_crtc, + const struct intel_crtc_state *cstate, const struct intel_wm_config *config, - const struct skl_pipe_wm_parameters *params, struct skl_ddb_entry *alloc /* out */) { + struct drm_crtc *for_crtc = cstate->base.crtc; struct drm_crtc *crtc; unsigned int pipe_size, ddb_size; int nth_active_pipe; - if (!params->active) { + if (!cstate->base.active) { alloc->start = 0; alloc->end = 0; return; @@ -2624,7 +2822,12 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, int plane; u32 val; + memset(ddb, 0, sizeof(*ddb)); + for_each_pipe(dev_priv, pipe) { + if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe))) + continue; + for_each_plane(dev_priv, pipe, plane) { val = I915_READ(PLANE_BUF_CFG(pipe, plane)); skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane], @@ -2632,24 +2835,35 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, } val = I915_READ(CUR_BUF_CFG(pipe)); - skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val); + skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR], + val); } } static unsigned int -skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y) +skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, + const struct drm_plane_state *pstate, + int y) { + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); + struct drm_framebuffer *fb = pstate->fb; /* for planar format */ - if (p->y_bytes_per_pixel) { + if (fb->pixel_format == DRM_FORMAT_NV12) { if (y) /* y-plane data rate */ - return p->horiz_pixels * p->vert_pixels * p->y_bytes_per_pixel; + return intel_crtc->config->pipe_src_w * + intel_crtc->config->pipe_src_h * + drm_format_plane_cpp(fb->pixel_format, 0); else /* uv-plane data rate */ - return (p->horiz_pixels/2) * (p->vert_pixels/2) * p->bytes_per_pixel; + return (intel_crtc->config->pipe_src_w/2) * + (intel_crtc->config->pipe_src_h/2) * + drm_format_plane_cpp(fb->pixel_format, 1); } /* for packed formats */ - return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel; + return intel_crtc->config->pipe_src_w * + intel_crtc->config->pipe_src_h * + drm_format_plane_cpp(fb->pixel_format, 0); } /* @@ -2658,72 +2872,85 @@ skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y) * 3 * 4096 * 8192 * 4 < 2^32 */ static unsigned int -skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc, - const struct skl_pipe_wm_parameters *params) +skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate) { + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); + struct drm_device *dev = intel_crtc->base.dev; + const struct intel_plane *intel_plane; unsigned int total_data_rate = 0; - int plane; - for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) { - const struct intel_plane_wm_parameters *p; + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + const struct drm_plane_state *pstate = intel_plane->base.state; - p = ¶ms->plane[plane]; - if (!p->enabled) + if (pstate->fb == NULL) continue; - total_data_rate += skl_plane_relative_data_rate(p, 0); /* packed/uv */ - if (p->y_bytes_per_pixel) { - total_data_rate += skl_plane_relative_data_rate(p, 1); /* y-plane */ - } + if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR) + continue; + + /* packed/uv */ + total_data_rate += skl_plane_relative_data_rate(cstate, + pstate, + 0); + + if (pstate->fb->pixel_format == DRM_FORMAT_NV12) + /* y-plane */ + total_data_rate += skl_plane_relative_data_rate(cstate, + pstate, + 1); } return total_data_rate; } static void -skl_allocate_pipe_ddb(struct drm_crtc *crtc, - const struct intel_wm_config *config, - const struct skl_pipe_wm_parameters *params, +skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, struct skl_ddb_allocation *ddb /* out */) { + struct drm_crtc *crtc = cstate->base.crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_wm_config *config = &dev_priv->wm.config; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_plane *intel_plane; enum pipe pipe = intel_crtc->pipe; struct skl_ddb_entry *alloc = &ddb->pipe[pipe]; uint16_t alloc_size, start, cursor_blocks; uint16_t minimum[I915_MAX_PLANES]; uint16_t y_minimum[I915_MAX_PLANES]; unsigned int total_data_rate; - int plane; - skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc); + skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc); alloc_size = skl_ddb_entry_size(alloc); if (alloc_size == 0) { memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); - memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe])); + memset(&ddb->plane[pipe][PLANE_CURSOR], 0, + sizeof(ddb->plane[pipe][PLANE_CURSOR])); return; } cursor_blocks = skl_cursor_allocation(config); - ddb->cursor[pipe].start = alloc->end - cursor_blocks; - ddb->cursor[pipe].end = alloc->end; + ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks; + ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; alloc_size -= cursor_blocks; alloc->end -= cursor_blocks; /* 1. Allocate the mininum required blocks for each active plane */ - for_each_plane(dev_priv, pipe, plane) { - const struct intel_plane_wm_parameters *p; + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + struct drm_plane *plane = &intel_plane->base; + struct drm_framebuffer *fb = plane->state->fb; + int id = skl_wm_plane_id(intel_plane); - p = ¶ms->plane[plane]; - if (!p->enabled) + if (fb == NULL) + continue; + if (plane->type == DRM_PLANE_TYPE_CURSOR) continue; - minimum[plane] = 8; - alloc_size -= minimum[plane]; - y_minimum[plane] = p->y_bytes_per_pixel ? 8 : 0; - alloc_size -= y_minimum[plane]; + minimum[id] = 8; + alloc_size -= minimum[id]; + y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0; + alloc_size -= y_minimum[id]; } /* @@ -2732,45 +2959,50 @@ skl_allocate_pipe_ddb(struct drm_crtc *crtc, * * FIXME: we may not allocate every single block here. */ - total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params); + total_data_rate = skl_get_total_relative_data_rate(cstate); start = alloc->start; - for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) { - const struct intel_plane_wm_parameters *p; + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + struct drm_plane *plane = &intel_plane->base; + struct drm_plane_state *pstate = intel_plane->base.state; unsigned int data_rate, y_data_rate; uint16_t plane_blocks, y_plane_blocks = 0; + int id = skl_wm_plane_id(intel_plane); - p = ¶ms->plane[plane]; - if (!p->enabled) + if (pstate->fb == NULL) + continue; + if (plane->type == DRM_PLANE_TYPE_CURSOR) continue; - data_rate = skl_plane_relative_data_rate(p, 0); + data_rate = skl_plane_relative_data_rate(cstate, pstate, 0); /* * allocation for (packed formats) or (uv-plane part of planar format): * promote the expression to 64 bits to avoid overflowing, the * result is < available as data_rate / total_data_rate < 1 */ - plane_blocks = minimum[plane]; + plane_blocks = minimum[id]; plane_blocks += div_u64((uint64_t)alloc_size * data_rate, total_data_rate); - ddb->plane[pipe][plane].start = start; - ddb->plane[pipe][plane].end = start + plane_blocks; + ddb->plane[pipe][id].start = start; + ddb->plane[pipe][id].end = start + plane_blocks; start += plane_blocks; /* * allocation for y_plane part of planar format: */ - if (p->y_bytes_per_pixel) { - y_data_rate = skl_plane_relative_data_rate(p, 1); - y_plane_blocks = y_minimum[plane]; + if (pstate->fb->pixel_format == DRM_FORMAT_NV12) { + y_data_rate = skl_plane_relative_data_rate(cstate, + pstate, + 1); + y_plane_blocks = y_minimum[id]; y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate, total_data_rate); - ddb->y_plane[pipe][plane].start = start; - ddb->y_plane[pipe][plane].end = start + y_plane_blocks; + ddb->y_plane[pipe][id].start = start; + ddb->y_plane[pipe][id].end = start + y_plane_blocks; start += y_plane_blocks; } @@ -2840,103 +3072,27 @@ static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb, struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; - enum pipe pipe = intel_crtc->pipe; - - if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe], - sizeof(new_ddb->plane[pipe]))) - return true; - if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe], - sizeof(new_ddb->cursor[pipe]))) + /* + * If ddb allocation of pipes changed, it may require recalculation of + * watermarks + */ + if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe))) return true; return false; } -static void skl_compute_wm_global_parameters(struct drm_device *dev, - struct intel_wm_config *config) -{ - struct drm_crtc *crtc; - struct drm_plane *plane; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) - config->num_pipes_active += to_intel_crtc(crtc)->active; - - /* FIXME: I don't think we need those two global parameters on SKL */ - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { - struct intel_plane *intel_plane = to_intel_plane(plane); - - config->sprites_enabled |= intel_plane->wm.enabled; - config->sprites_scaled |= intel_plane->wm.scaled; - } -} - -static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc, - struct skl_pipe_wm_parameters *p) -{ - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; - struct drm_plane *plane; - struct drm_framebuffer *fb; - int i = 1; /* Index for sprite planes start */ - - p->active = intel_crtc->active; - if (p->active) { - p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal; - p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config); - - fb = crtc->primary->state->fb; - /* For planar: Bpp is for uv plane, y_Bpp is for y plane */ - if (fb) { - p->plane[0].enabled = true; - p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ? - drm_format_plane_cpp(fb->pixel_format, 1) : fb->bits_per_pixel / 8; - p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ? - drm_format_plane_cpp(fb->pixel_format, 0) : 0; - p->plane[0].tiling = fb->modifier[0]; - } else { - p->plane[0].enabled = false; - p->plane[0].bytes_per_pixel = 0; - p->plane[0].y_bytes_per_pixel = 0; - p->plane[0].tiling = DRM_FORMAT_MOD_NONE; - } - p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w; - p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h; - p->plane[0].rotation = crtc->primary->state->rotation; - - fb = crtc->cursor->state->fb; - p->cursor.y_bytes_per_pixel = 0; - if (fb) { - p->cursor.enabled = true; - p->cursor.bytes_per_pixel = fb->bits_per_pixel / 8; - p->cursor.horiz_pixels = crtc->cursor->state->crtc_w; - p->cursor.vert_pixels = crtc->cursor->state->crtc_h; - } else { - p->cursor.enabled = false; - p->cursor.bytes_per_pixel = 0; - p->cursor.horiz_pixels = 64; - p->cursor.vert_pixels = 64; - } - } - - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { - struct intel_plane *intel_plane = to_intel_plane(plane); - - if (intel_plane->pipe == pipe && - plane->type == DRM_PLANE_TYPE_OVERLAY) - p->plane[i++] = intel_plane->wm; - } -} - static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, - struct skl_pipe_wm_parameters *p, - struct intel_plane_wm_parameters *p_params, + struct intel_crtc_state *cstate, + struct intel_plane *intel_plane, uint16_t ddb_allocation, int level, uint16_t *out_blocks, /* out */ uint8_t *out_lines /* out */) { + struct drm_plane *plane = &intel_plane->base; + struct drm_framebuffer *fb = plane->state->fb; uint32_t latency = dev_priv->wm.skl_latency[level]; uint32_t method1, method2; uint32_t plane_bytes_per_line, plane_blocks_per_line; @@ -2944,31 +3100,33 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, uint32_t selected_result; uint8_t bytes_per_pixel; - if (latency == 0 || !p->active || !p_params->enabled) + if (latency == 0 || !cstate->base.active || !fb) return false; - bytes_per_pixel = p_params->y_bytes_per_pixel ? - p_params->y_bytes_per_pixel : - p_params->bytes_per_pixel; - method1 = skl_wm_method1(p->pixel_rate, + bytes_per_pixel = drm_format_plane_cpp(fb->pixel_format, 0); + method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate), bytes_per_pixel, latency); - method2 = skl_wm_method2(p->pixel_rate, - p->pipe_htotal, - p_params->horiz_pixels, + method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate), + cstate->base.adjusted_mode.crtc_htotal, + cstate->pipe_src_w, bytes_per_pixel, - p_params->tiling, + fb->modifier[0], latency); - plane_bytes_per_line = p_params->horiz_pixels * bytes_per_pixel; + plane_bytes_per_line = cstate->pipe_src_w * bytes_per_pixel; plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512); - if (p_params->tiling == I915_FORMAT_MOD_Y_TILED || - p_params->tiling == I915_FORMAT_MOD_Yf_TILED) { + if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || + fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { uint32_t min_scanlines = 4; uint32_t y_tile_minimum; - if (intel_rotation_90_or_270(p_params->rotation)) { - switch (p_params->bytes_per_pixel) { + if (intel_rotation_90_or_270(plane->state->rotation)) { + int bpp = (fb->pixel_format == DRM_FORMAT_NV12) ? + drm_format_plane_cpp(fb->pixel_format, 1) : + drm_format_plane_cpp(fb->pixel_format, 0); + + switch (bpp) { case 1: min_scanlines = 16; break; @@ -2992,8 +3150,8 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line); if (level >= 1 && level <= 7) { - if (p_params->tiling == I915_FORMAT_MOD_Y_TILED || - p_params->tiling == I915_FORMAT_MOD_Yf_TILED) + if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || + fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) res_lines += 4; else res_blocks++; @@ -3010,81 +3168,80 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, static void skl_compute_wm_level(const struct drm_i915_private *dev_priv, struct skl_ddb_allocation *ddb, - struct skl_pipe_wm_parameters *p, - enum pipe pipe, + struct intel_crtc_state *cstate, int level, - int num_planes, struct skl_wm_level *result) { + struct drm_device *dev = dev_priv->dev; + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); + struct intel_plane *intel_plane; uint16_t ddb_blocks; - int i; + enum pipe pipe = intel_crtc->pipe; + + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + int i = skl_wm_plane_id(intel_plane); - for (i = 0; i < num_planes; i++) { ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]); result->plane_en[i] = skl_compute_plane_wm(dev_priv, - p, &p->plane[i], + cstate, + intel_plane, ddb_blocks, level, &result->plane_res_b[i], &result->plane_res_l[i]); } - - ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]); - result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor, - ddb_blocks, level, - &result->cursor_res_b, - &result->cursor_res_l); } static uint32_t -skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p) +skl_compute_linetime_wm(struct intel_crtc_state *cstate) { - if (!to_intel_crtc(crtc)->active) + if (!cstate->base.active) return 0; - return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate); + if (WARN_ON(skl_pipe_pixel_rate(cstate) == 0)) + return 0; + return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000, + skl_pipe_pixel_rate(cstate)); } -static void skl_compute_transition_wm(struct drm_crtc *crtc, - struct skl_pipe_wm_parameters *params, +static void skl_compute_transition_wm(struct intel_crtc_state *cstate, struct skl_wm_level *trans_wm /* out */) { + struct drm_crtc *crtc = cstate->base.crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int i; + struct intel_plane *intel_plane; - if (!params->active) + if (!cstate->base.active) return; /* Until we know more, just disable transition WMs */ - for (i = 0; i < intel_num_planes(intel_crtc); i++) + for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) { + int i = skl_wm_plane_id(intel_plane); + trans_wm->plane_en[i] = false; - trans_wm->cursor_en = false; + } } -static void skl_compute_pipe_wm(struct drm_crtc *crtc, +static void skl_compute_pipe_wm(struct intel_crtc_state *cstate, struct skl_ddb_allocation *ddb, - struct skl_pipe_wm_parameters *params, struct skl_pipe_wm *pipe_wm) { - struct drm_device *dev = crtc->dev; + struct drm_device *dev = cstate->base.crtc->dev; const struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int level, max_level = ilk_wm_max_level(dev); for (level = 0; level <= max_level; level++) { - skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe, - level, intel_num_planes(intel_crtc), - &pipe_wm->wm[level]); + skl_compute_wm_level(dev_priv, ddb, cstate, + level, &pipe_wm->wm[level]); } - pipe_wm->linetime = skl_compute_linetime_wm(crtc, params); + pipe_wm->linetime = skl_compute_linetime_wm(cstate); - skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm); + skl_compute_transition_wm(cstate, &pipe_wm->trans_wm); } static void skl_compute_wm_results(struct drm_device *dev, - struct skl_pipe_wm_parameters *p, struct skl_pipe_wm *p_wm, struct skl_wm_values *r, struct intel_crtc *intel_crtc) @@ -3109,13 +3266,13 @@ static void skl_compute_wm_results(struct drm_device *dev, temp = 0; - temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT; - temp |= p_wm->wm[level].cursor_res_b; + temp |= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT; + temp |= p_wm->wm[level].plane_res_b[PLANE_CURSOR]; - if (p_wm->wm[level].cursor_en) + if (p_wm->wm[level].plane_en[PLANE_CURSOR]) temp |= PLANE_WM_EN; - r->cursor[pipe][level] = temp; + r->plane[pipe][PLANE_CURSOR][level] = temp; } @@ -3131,17 +3288,18 @@ static void skl_compute_wm_results(struct drm_device *dev, } temp = 0; - temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT; - temp |= p_wm->trans_wm.cursor_res_b; - if (p_wm->trans_wm.cursor_en) + temp |= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT; + temp |= p_wm->trans_wm.plane_res_b[PLANE_CURSOR]; + if (p_wm->trans_wm.plane_en[PLANE_CURSOR]) temp |= PLANE_WM_EN; - r->cursor_trans[pipe] = temp; + r->plane_trans[pipe][PLANE_CURSOR] = temp; r->wm_linetime[pipe] = p_wm->linetime; } -static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg, +static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, + i915_reg_t reg, const struct skl_ddb_entry *entry) { if (entry->end) @@ -3170,12 +3328,13 @@ static void skl_write_wm_values(struct drm_i915_private *dev_priv, I915_WRITE(PLANE_WM(pipe, i, level), new->plane[pipe][i][level]); I915_WRITE(CUR_WM(pipe, level), - new->cursor[pipe][level]); + new->plane[pipe][PLANE_CURSOR][level]); } for (i = 0; i < intel_num_planes(crtc); i++) I915_WRITE(PLANE_WM_TRANS(pipe, i), new->plane_trans[pipe][i]); - I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]); + I915_WRITE(CUR_WM_TRANS(pipe), + new->plane_trans[pipe][PLANE_CURSOR]); for (i = 0; i < intel_num_planes(crtc); i++) { skl_ddb_entry_write(dev_priv, @@ -3187,7 +3346,7 @@ static void skl_write_wm_values(struct drm_i915_private *dev_priv, } skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), - &new->ddb.cursor[pipe]); + &new->ddb.plane[pipe][PLANE_CURSOR]); } } @@ -3327,28 +3486,25 @@ static void skl_flush_wm_values(struct drm_i915_private *dev_priv, } static bool skl_update_pipe_wm(struct drm_crtc *crtc, - struct skl_pipe_wm_parameters *params, - struct intel_wm_config *config, struct skl_ddb_allocation *ddb, /* out */ struct skl_pipe_wm *pipe_wm /* out */) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - skl_compute_wm_pipe_parameters(crtc, params); - skl_allocate_pipe_ddb(crtc, config, params, ddb); - skl_compute_pipe_wm(crtc, ddb, params, pipe_wm); + skl_allocate_pipe_ddb(cstate, ddb); + skl_compute_pipe_wm(cstate, ddb, pipe_wm); - if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm))) + if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm))) return false; - intel_crtc->wm.skl_active = *pipe_wm; + intel_crtc->wm.active.skl = *pipe_wm; return true; } static void skl_update_other_pipe_wm(struct drm_device *dev, struct drm_crtc *crtc, - struct intel_wm_config *config, struct skl_wm_values *r) { struct intel_crtc *intel_crtc; @@ -3369,7 +3525,6 @@ static void skl_update_other_pipe_wm(struct drm_device *dev, */ list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) { - struct skl_pipe_wm_parameters params = {}; struct skl_pipe_wm pipe_wm = {}; bool wm_changed; @@ -3380,7 +3535,6 @@ static void skl_update_other_pipe_wm(struct drm_device *dev, continue; wm_changed = skl_update_pipe_wm(&intel_crtc->base, - ¶ms, config, &r->ddb, &pipe_wm); /* @@ -3390,33 +3544,53 @@ static void skl_update_other_pipe_wm(struct drm_device *dev, */ WARN_ON(!wm_changed); - skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc); + skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc); r->dirty[intel_crtc->pipe] = true; } } +static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe) +{ + watermarks->wm_linetime[pipe] = 0; + memset(watermarks->plane[pipe], 0, + sizeof(uint32_t) * 8 * I915_MAX_PLANES); + memset(watermarks->plane_trans[pipe], + 0, sizeof(uint32_t) * I915_MAX_PLANES); + watermarks->plane_trans[pipe][PLANE_CURSOR] = 0; + + /* Clear ddb entries for pipe */ + memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry)); + memset(&watermarks->ddb.plane[pipe], 0, + sizeof(struct skl_ddb_entry) * I915_MAX_PLANES); + memset(&watermarks->ddb.y_plane[pipe], 0, + sizeof(struct skl_ddb_entry) * I915_MAX_PLANES); + memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0, + sizeof(struct skl_ddb_entry)); + +} + static void skl_update_wm(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct skl_pipe_wm_parameters params = {}; struct skl_wm_values *results = &dev_priv->wm.skl_results; - struct skl_pipe_wm pipe_wm = {}; - struct intel_wm_config config = {}; + struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl; - memset(results, 0, sizeof(*results)); - skl_compute_wm_global_parameters(dev, &config); + /* Clear all dirty flags */ + memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES); - if (!skl_update_pipe_wm(crtc, ¶ms, &config, - &results->ddb, &pipe_wm)) + skl_clear_wm(results, intel_crtc->pipe); + + if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm)) return; - skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc); + skl_compute_wm_results(dev, pipe_wm, results, intel_crtc); results->dirty[intel_crtc->pipe] = true; - skl_update_other_pipe_wm(dev, crtc, &config, results); + skl_update_other_pipe_wm(dev, crtc, results); skl_write_wm_values(dev_priv, results); skl_flush_wm_values(dev_priv, results); @@ -3424,71 +3598,23 @@ static void skl_update_wm(struct drm_crtc *crtc) dev_priv->wm.skl_hw = *results; } -static void -skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc, - uint32_t sprite_width, uint32_t sprite_height, - int pixel_size, bool enabled, bool scaled) +static void ilk_program_watermarks(struct drm_i915_private *dev_priv) { - struct intel_plane *intel_plane = to_intel_plane(plane); - struct drm_framebuffer *fb = plane->state->fb; - - intel_plane->wm.enabled = enabled; - intel_plane->wm.scaled = scaled; - intel_plane->wm.horiz_pixels = sprite_width; - intel_plane->wm.vert_pixels = sprite_height; - intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE; - - /* For planar: Bpp is for UV plane, y_Bpp is for Y plane */ - intel_plane->wm.bytes_per_pixel = - (fb && fb->pixel_format == DRM_FORMAT_NV12) ? - drm_format_plane_cpp(plane->state->fb->pixel_format, 1) : pixel_size; - intel_plane->wm.y_bytes_per_pixel = - (fb && fb->pixel_format == DRM_FORMAT_NV12) ? - drm_format_plane_cpp(plane->state->fb->pixel_format, 0) : 0; - - /* - * Framebuffer can be NULL on plane disable, but it does not - * matter for watermarks if we assume no tiling in that case. - */ - if (fb) - intel_plane->wm.tiling = fb->modifier[0]; - intel_plane->wm.rotation = plane->state->rotation; - - skl_update_wm(crtc); -} - -static void ilk_update_wm(struct drm_crtc *crtc) -{ - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_device *dev = dev_priv->dev; + struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; struct ilk_wm_maximums max; - struct ilk_pipe_wm_parameters params = {}; + struct intel_wm_config *config = &dev_priv->wm.config; struct ilk_wm_values results = {}; enum intel_ddb_partitioning partitioning; - struct intel_pipe_wm pipe_wm = {}; - struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; - struct intel_wm_config config = {}; - - ilk_compute_wm_parameters(crtc, ¶ms); - intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm); - - if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm))) - return; - - intel_crtc->wm.active = pipe_wm; - - ilk_compute_wm_config(dev, &config); - - ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max); - ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); + ilk_compute_wm_maximums(dev, 1, config, INTEL_DDB_PART_1_2, &max); + ilk_wm_merge(dev, config, &max, &lp_wm_1_2); /* 5/6 split only in single pipe config on IVB+ */ if (INTEL_INFO(dev)->gen >= 7 && - config.num_pipes_active == 1 && config.sprites_enabled) { - ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); - ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); + config->num_pipes_active == 1 && config->sprites_enabled) { + ilk_compute_wm_maximums(dev, 1, config, INTEL_DDB_PART_5_6, &max); + ilk_wm_merge(dev, config, &max, &lp_wm_5_6); best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6); } else { @@ -3503,20 +3629,13 @@ static void ilk_update_wm(struct drm_crtc *crtc) ilk_write_wm_values(dev_priv, &results); } -static void -ilk_update_sprite_wm(struct drm_plane *plane, - struct drm_crtc *crtc, - uint32_t sprite_width, uint32_t sprite_height, - int pixel_size, bool enabled, bool scaled) +static void ilk_update_wm(struct drm_crtc *crtc) { - struct drm_device *dev = plane->dev; - struct intel_plane *intel_plane = to_intel_plane(plane); + struct drm_i915_private *dev_priv = to_i915(crtc->dev); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - intel_plane->wm.enabled = enabled; - intel_plane->wm.scaled = scaled; - intel_plane->wm.horiz_pixels = sprite_width; - intel_plane->wm.vert_pixels = sprite_width; - intel_plane->wm.bytes_per_pixel = pixel_size; + WARN_ON(cstate->base.active != intel_crtc->active); /* * IVB workaround: must disable low power watermarks for at least @@ -3525,10 +3644,14 @@ ilk_update_sprite_wm(struct drm_plane *plane, * * WaCxSRDisabledForSpriteScaling:ivb */ - if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev)) - intel_wait_for_vblank(dev, intel_plane->pipe); + if (cstate->disable_lp_wm) { + ilk_disable_lp_wm(crtc->dev); + intel_wait_for_vblank(crtc->dev, intel_crtc->pipe); + } + + intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk; - ilk_update_wm(crtc); + ilk_program_watermarks(dev_priv); } static void skl_pipe_wm_active_state(uint32_t val, @@ -3549,10 +3672,10 @@ static void skl_pipe_wm_active_state(uint32_t val, (val >> PLANE_WM_LINES_SHIFT) & PLANE_WM_LINES_MASK; } else { - active->wm[level].cursor_en = is_enabled; - active->wm[level].cursor_res_b = + active->wm[level].plane_en[PLANE_CURSOR] = is_enabled; + active->wm[level].plane_res_b[PLANE_CURSOR] = val & PLANE_WM_BLOCKS_MASK; - active->wm[level].cursor_res_l = + active->wm[level].plane_res_l[PLANE_CURSOR] = (val >> PLANE_WM_LINES_SHIFT) & PLANE_WM_LINES_MASK; } @@ -3565,10 +3688,10 @@ static void skl_pipe_wm_active_state(uint32_t val, (val >> PLANE_WM_LINES_SHIFT) & PLANE_WM_LINES_MASK; } else { - active->trans_wm.cursor_en = is_enabled; - active->trans_wm.cursor_res_b = + active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled; + active->trans_wm.plane_res_b[PLANE_CURSOR] = val & PLANE_WM_BLOCKS_MASK; - active->trans_wm.cursor_res_l = + active->trans_wm.plane_res_l[PLANE_CURSOR] = (val >> PLANE_WM_LINES_SHIFT) & PLANE_WM_LINES_MASK; } @@ -3581,7 +3704,8 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct skl_wm_values *hw = &dev_priv->wm.skl_hw; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct skl_pipe_wm *active = &intel_crtc->wm.skl_active; + struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct skl_pipe_wm *active = &cstate->wm.optimal.skl; enum pipe pipe = intel_crtc->pipe; int level, i, max_level; uint32_t temp; @@ -3594,12 +3718,12 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) for (i = 0; i < intel_num_planes(intel_crtc); i++) hw->plane[pipe][i][level] = I915_READ(PLANE_WM(pipe, i, level)); - hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level)); + hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level)); } for (i = 0; i < intel_num_planes(intel_crtc); i++) hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i)); - hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe)); + hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe)); if (!intel_crtc->active) return; @@ -3614,7 +3738,7 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) skl_pipe_wm_active_state(temp, active, false, false, i, level); } - temp = hw->cursor[pipe][level]; + temp = hw->plane[pipe][PLANE_CURSOR][level]; skl_pipe_wm_active_state(temp, active, false, true, i, level); } @@ -3623,8 +3747,10 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) skl_pipe_wm_active_state(temp, active, true, false, i, 0); } - temp = hw->cursor_trans[pipe]; + temp = hw->plane_trans[pipe][PLANE_CURSOR]; skl_pipe_wm_active_state(temp, active, true, true, i, 0); + + intel_crtc->wm.active.skl = *active; } void skl_wm_get_hw_state(struct drm_device *dev) @@ -3644,9 +3770,10 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct ilk_wm_values *hw = &dev_priv->wm.hw; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_pipe_wm *active = &intel_crtc->wm.active; + struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct intel_pipe_wm *active = &cstate->wm.optimal.ilk; enum pipe pipe = intel_crtc->pipe; - static const unsigned int wm0_pipe_reg[] = { + static const i915_reg_t wm0_pipe_reg[] = { [PIPE_A] = WM0_PIPEA_ILK, [PIPE_B] = WM0_PIPEB_ILK, [PIPE_C] = WM0_PIPEC_IVB, @@ -3683,6 +3810,161 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) for (level = 0; level <= max_level; level++) active->wm[level].enable = true; } + + intel_crtc->wm.active.ilk = *active; +} + +#define _FW_WM(value, plane) \ + (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT) +#define _FW_WM_VLV(value, plane) \ + (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT) + +static void vlv_read_wm_values(struct drm_i915_private *dev_priv, + struct vlv_wm_values *wm) +{ + enum pipe pipe; + uint32_t tmp; + + for_each_pipe(dev_priv, pipe) { + tmp = I915_READ(VLV_DDL(pipe)); + + wm->ddl[pipe].primary = + (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + wm->ddl[pipe].cursor = + (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + wm->ddl[pipe].sprite[0] = + (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + wm->ddl[pipe].sprite[1] = + (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + } + + tmp = I915_READ(DSPFW1); + wm->sr.plane = _FW_WM(tmp, SR); + wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB); + wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB); + wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA); + + tmp = I915_READ(DSPFW2); + wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB); + wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA); + wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA); + + tmp = I915_READ(DSPFW3); + wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); + + if (IS_CHERRYVIEW(dev_priv)) { + tmp = I915_READ(DSPFW7_CHV); + wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED); + wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC); + + tmp = I915_READ(DSPFW8_CHV); + wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF); + wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE); + + tmp = I915_READ(DSPFW9_CHV); + wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC); + wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC); + + tmp = I915_READ(DSPHOWM); + wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; + wm->pipe[PIPE_C].sprite[1] |= _FW_WM(tmp, SPRITEF_HI) << 8; + wm->pipe[PIPE_C].sprite[0] |= _FW_WM(tmp, SPRITEE_HI) << 8; + wm->pipe[PIPE_C].primary |= _FW_WM(tmp, PLANEC_HI) << 8; + wm->pipe[PIPE_B].sprite[1] |= _FW_WM(tmp, SPRITED_HI) << 8; + wm->pipe[PIPE_B].sprite[0] |= _FW_WM(tmp, SPRITEC_HI) << 8; + wm->pipe[PIPE_B].primary |= _FW_WM(tmp, PLANEB_HI) << 8; + wm->pipe[PIPE_A].sprite[1] |= _FW_WM(tmp, SPRITEB_HI) << 8; + wm->pipe[PIPE_A].sprite[0] |= _FW_WM(tmp, SPRITEA_HI) << 8; + wm->pipe[PIPE_A].primary |= _FW_WM(tmp, PLANEA_HI) << 8; + } else { + tmp = I915_READ(DSPFW7); + wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED); + wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC); + + tmp = I915_READ(DSPHOWM); + wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; + wm->pipe[PIPE_B].sprite[1] |= _FW_WM(tmp, SPRITED_HI) << 8; + wm->pipe[PIPE_B].sprite[0] |= _FW_WM(tmp, SPRITEC_HI) << 8; + wm->pipe[PIPE_B].primary |= _FW_WM(tmp, PLANEB_HI) << 8; + wm->pipe[PIPE_A].sprite[1] |= _FW_WM(tmp, SPRITEB_HI) << 8; + wm->pipe[PIPE_A].sprite[0] |= _FW_WM(tmp, SPRITEA_HI) << 8; + wm->pipe[PIPE_A].primary |= _FW_WM(tmp, PLANEA_HI) << 8; + } +} + +#undef _FW_WM +#undef _FW_WM_VLV + +void vlv_wm_get_hw_state(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct vlv_wm_values *wm = &dev_priv->wm.vlv; + struct intel_plane *plane; + enum pipe pipe; + u32 val; + + vlv_read_wm_values(dev_priv, wm); + + for_each_intel_plane(dev, plane) { + switch (plane->base.type) { + int sprite; + case DRM_PLANE_TYPE_CURSOR: + plane->wm.fifo_size = 63; + break; + case DRM_PLANE_TYPE_PRIMARY: + plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0); + break; + case DRM_PLANE_TYPE_OVERLAY: + sprite = plane->plane; + plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1); + break; + } + } + + wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; + wm->level = VLV_WM_LEVEL_PM2; + + if (IS_CHERRYVIEW(dev_priv)) { + mutex_lock(&dev_priv->rps.hw_lock); + + val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ); + if (val & DSP_MAXFIFO_PM5_ENABLE) + wm->level = VLV_WM_LEVEL_PM5; + + /* + * If DDR DVFS is disabled in the BIOS, Punit + * will never ack the request. So if that happens + * assume we don't have to enable/disable DDR DVFS + * dynamically. To test that just set the REQ_ACK + * bit to poke the Punit, but don't change the + * HIGH/LOW bits so that we don't actually change + * the current state. + */ + val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); + val |= FORCE_DDR_FREQ_REQ_ACK; + vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val); + + if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) & + FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) { + DRM_DEBUG_KMS("Punit not acking DDR DVFS request, " + "assuming DDR DVFS is disabled\n"); + dev_priv->wm.max_level = VLV_WM_LEVEL_PM5; + } else { + val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); + if ((val & FORCE_DDR_HIGH_FREQ) == 0) + wm->level = VLV_WM_LEVEL_DDR_DVFS; + } + + mutex_unlock(&dev_priv->rps.hw_lock); + } + + for_each_pipe(dev_priv, pipe) + DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n", + pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor, + wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]); + + DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n", + wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr); } void ilk_wm_get_hw_state(struct drm_device *dev) @@ -3755,21 +4037,6 @@ void intel_update_watermarks(struct drm_crtc *crtc) dev_priv->display.update_wm(crtc); } -void intel_update_sprite_watermarks(struct drm_plane *plane, - struct drm_crtc *crtc, - uint32_t sprite_width, - uint32_t sprite_height, - int pixel_size, - bool enabled, bool scaled) -{ - struct drm_i915_private *dev_priv = plane->dev->dev_private; - - if (dev_priv->display.update_sprite_wm) - dev_priv->display.update_sprite_wm(plane, crtc, - sprite_width, sprite_height, - pixel_size, enabled, scaled); -} - /** * Lock protecting IPS related data structures */ @@ -3831,7 +4098,7 @@ static void ironlake_enable_drps(struct drm_device *dev) fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT; - vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >> + vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; dev_priv->ips.fmax = fmax; /* IPS callback will increase this */ @@ -3862,10 +4129,10 @@ static void ironlake_enable_drps(struct drm_device *dev) ironlake_set_drps(dev, fstart); - dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) + - I915_READ(0x112e0); + dev_priv->ips.last_count1 = I915_READ(DMIEC) + + I915_READ(DDREC) + I915_READ(CSIEC); dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); - dev_priv->ips.last_count2 = I915_READ(0x112f4); + dev_priv->ips.last_count2 = I915_READ(GFXEC); dev_priv->ips.last_time2 = ktime_get_raw_ns(); spin_unlock_irq(&mchdev_lock); @@ -4036,6 +4303,10 @@ static void gen6_set_rps(struct drm_device *dev, u8 val) { struct drm_i915_private *dev_priv = dev->dev_private; + /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + return; + WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); WARN_ON(val > dev_priv->rps.max_freq); WARN_ON(val < dev_priv->rps.min_freq); @@ -4068,7 +4339,7 @@ static void gen6_set_rps(struct drm_device *dev, u8 val) POSTING_READ(GEN6_RPNSWREQ); dev_priv->rps.cur_freq = val; - trace_intel_gpu_freq_change(val * 50); + trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); } static void valleyview_set_rps(struct drm_device *dev, u8 val) @@ -4083,14 +4354,14 @@ static void valleyview_set_rps(struct drm_device *dev, u8 val) "Odd GPU freq value\n")) val &= ~1; + I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); + if (val != dev_priv->rps.cur_freq) { vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); if (!IS_CHERRYVIEW(dev_priv)) gen6_set_rps_thresholds(dev_priv, val); } - I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); - dev_priv->rps.cur_freq = val; trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); } @@ -4250,12 +4521,8 @@ static void intel_print_rc6_info(struct drm_device *dev, u32 mode) static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) { - /* No RC6 before Ironlake */ - if (INTEL_INFO(dev)->gen < 5) - return 0; - - /* RC6 is only on Ironlake mobile not on desktop */ - if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev)) + /* No RC6 before Ironlake and code is gone for ilk. */ + if (INTEL_INFO(dev)->gen < 6) return 0; /* Respect the kernel parameter if it is set */ @@ -4275,10 +4542,6 @@ static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) return enable_rc6 & mask; } - /* Disable RC6 on Ironlake */ - if (INTEL_INFO(dev)->gen == 5) - return 0; - if (IS_IVYBRIDGE(dev)) return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); @@ -4297,25 +4560,27 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) u32 ddcc_status = 0; int ret; - rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); /* All of these values are in units of 50MHz */ dev_priv->rps.cur_freq = 0; /* static values from HW: RP0 > RP1 > RPn (min_freq) */ - dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; - dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; - dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; - if (IS_SKYLAKE(dev)) { - /* Store the frequency values in 16.66 MHZ units, which is - the natural hardware unit for SKL */ - dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; - dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER; - dev_priv->rps.min_freq *= GEN9_FREQ_SCALER; + if (IS_BROXTON(dev)) { + rp_state_cap = I915_READ(BXT_RP_STATE_CAP); + dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff; + dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; + dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff; + } else { + rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; + dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; + dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; } + /* hw_max = RP0 until we check for overclocking */ dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + if (IS_HASWELL(dev) || IS_BROADWELL(dev) || + IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { ret = sandybridge_pcode_read(dev_priv, HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, &ddcc_status); @@ -4327,6 +4592,16 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) dev_priv->rps.max_freq); } + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + /* Store the frequency values in 16.66 MHZ units, which is + the natural hardware unit for SKL */ + dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; + dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER; + dev_priv->rps.min_freq *= GEN9_FREQ_SCALER; + dev_priv->rps.max_freq *= GEN9_FREQ_SCALER; + dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER; + } + dev_priv->rps.idle_freq = dev_priv->rps.min_freq; /* Preserve min/max settings in case of re-init */ @@ -4353,6 +4628,12 @@ static void gen9_enable_rps(struct drm_device *dev) gen6_init_rps_frequencies(dev); + /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + return; + } + /* Program defaults and thresholds for RPS*/ I915_WRITE(GEN6_RC_VIDEO_FREQ, GEN9_FREQUENCY(dev_priv->rps.rp1_freq)); @@ -4390,13 +4671,22 @@ static void gen9_enable_rc6(struct drm_device *dev) I915_WRITE(GEN6_RC_CONTROL, 0); /* 2b: Program RC6 thresholds.*/ - I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); + + /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */ + if (IS_SKYLAKE(dev) && !((IS_SKL_GT3(dev) || IS_SKL_GT4(dev)) && + IS_SKL_REVID(dev, 0, SKL_REVID_E0))) + I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); + else + I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ for_each_ring(ring, dev_priv, unused) I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + + if (HAS_GUC_UCODE(dev)) + I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA); + I915_WRITE(GEN6_RC_SLEEP, 0); - I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ /* 2c: Program Coarse Power Gating Policies. */ I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25); @@ -4407,17 +4697,31 @@ static void gen9_enable_rc6(struct drm_device *dev) rc6_mask = GEN6_RC_CTL_RC6_ENABLE; DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off"); - I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | - GEN6_RC_CTL_EI_MODE(1) | - rc6_mask); + /* WaRsUseTimeoutMode */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */ + I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | + GEN7_RC_CTL_TO_MODE | + rc6_mask); + } else { + I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ + I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | + GEN6_RC_CTL_EI_MODE(1) | + rc6_mask); + } /* * 3b: Enable Coarse Power Gating only when RC6 is enabled. - * WaDisableRenderPowerGating:skl,bxt - Render PG need to be disabled with RC6. + * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6. */ - I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? - GEN9_MEDIA_PG_ENABLE : 0); - + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1) || + ((IS_SKL_GT3(dev) || IS_SKL_GT4(dev)) && + IS_SKL_REVID(dev, 0, SKL_REVID_E0))) + I915_WRITE(GEN9_PG_ENABLE, 0); + else + I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? + (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); @@ -4619,6 +4923,7 @@ static void __gen6_update_ring_freq(struct drm_device *dev) int min_freq = 15; unsigned int gpu_freq; unsigned int max_ia_freq, min_ring_freq; + unsigned int max_gpu_freq, min_gpu_freq; int scaling_factor = 180; struct cpufreq_policy *policy; @@ -4643,17 +4948,31 @@ static void __gen6_update_ring_freq(struct drm_device *dev) /* convert DDR frequency from units of 266.6MHz to bandwidth */ min_ring_freq = mult_frac(min_ring_freq, 8, 3); + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + /* Convert GT frequency to 50 HZ units */ + min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER; + max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER; + } else { + min_gpu_freq = dev_priv->rps.min_freq; + max_gpu_freq = dev_priv->rps.max_freq; + } + /* * For each potential GPU frequency, load a ring frequency we'd like * to use for memory access. We do this by specifying the IA frequency * the PCU should use as a reference to determine the ring frequency. */ - for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq; - gpu_freq--) { - int diff = dev_priv->rps.max_freq - gpu_freq; + for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) { + int diff = max_gpu_freq - gpu_freq; unsigned int ia_freq = 0, ring_freq = 0; - if (INTEL_INFO(dev)->gen >= 8) { + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + /* + * ring_freq = 2 * GT. ring_freq is in 100MHz units + * No floor required for ring frequency on SKL. + */ + ring_freq = gpu_freq; + } else if (INTEL_INFO(dev)->gen >= 8) { /* max(2 * GT, DDR). NB: GT is 50MHz units */ ring_freq = max(min_ring_freq, gpu_freq); } else if (IS_HASWELL(dev)) { @@ -4687,7 +5006,7 @@ void gen6_update_ring_freq(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev)) + if (!HAS_CORE_RING_FREQ(dev)) return; mutex_lock(&dev_priv->rps.hw_lock); @@ -4700,32 +5019,27 @@ static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) struct drm_device *dev = dev_priv->dev; u32 val, rp0; - if (dev->pdev->revision >= 0x20) { - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); + val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); - switch (INTEL_INFO(dev)->eu_total) { - case 8: - /* (2 * 4) config */ - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); - break; - case 12: - /* (2 * 6) config */ - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT); - break; - case 16: - /* (2 * 8) config */ - default: - /* Setting (2 * 8) Min RP0 for any other combination */ - rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT); - break; - } - rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK); - } else { - /* For pre-production hardware */ - val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG); - rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & - PUNIT_GPU_STATUS_MAX_FREQ_MASK; + switch (INTEL_INFO(dev)->eu_total) { + case 8: + /* (2 * 4) config */ + rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); + break; + case 12: + /* (2 * 6) config */ + rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT); + break; + case 16: + /* (2 * 8) config */ + default: + /* Setting (2 * 8) Min RP0 for any other combination */ + rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT); + break; } + + rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK); + return rp0; } @@ -4741,18 +5055,11 @@ static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv) static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; u32 val, rp1; - if (dev->pdev->revision >= 0x20) { - val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); - rp1 = (val & FB_GFX_FREQ_FUSE_MASK); - } else { - /* For pre-production hardware */ - val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); - rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & - PUNIT_GPU_STATUS_MAX_FREQ_MASK); - } + val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); + rp1 = (val & FB_GFX_FREQ_FUSE_MASK); + return rp1; } @@ -4967,25 +5274,10 @@ static void cherryview_init_gt_powersave(struct drm_device *dev) mutex_unlock(&dev_priv->sb_lock); switch ((val >> 2) & 0x7) { - case 0: - case 1: - dev_priv->rps.cz_freq = 200; - dev_priv->mem_freq = 1600; - break; - case 2: - dev_priv->rps.cz_freq = 267; - dev_priv->mem_freq = 1600; - break; case 3: - dev_priv->rps.cz_freq = 333; dev_priv->mem_freq = 2000; break; - case 4: - dev_priv->rps.cz_freq = 320; - dev_priv->mem_freq = 1600; - break; - case 5: - dev_priv->rps.cz_freq = 400; + default: dev_priv->mem_freq = 1600; break; } @@ -5117,7 +5409,7 @@ static void cherryview_enable_rps(struct drm_device *dev) /* RPS code assumes GPLL is used */ WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no"); + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); dev_priv->rps.cur_freq = (val >> 8) & 0xff; @@ -5207,7 +5499,7 @@ static void valleyview_enable_rps(struct drm_device *dev) /* RPS code assumes GPLL is used */ WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n"); - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no"); + DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); dev_priv->rps.cur_freq = (val >> 8) & 0xff; @@ -5416,7 +5708,7 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) assert_spin_locked(&mchdev_lock); - pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4)); + pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq)); pxvid = (pxvid >> 24) & 0x7f; ext_v = pvid_to_extvid(dev_priv, pxvid); @@ -5659,13 +5951,13 @@ static void intel_init_emon(struct drm_device *dev) I915_WRITE(CSIEW2, 0x04000004); for (i = 0; i < 5; i++) - I915_WRITE(PEW + (i * 4), 0); + I915_WRITE(PEW(i), 0); for (i = 0; i < 3; i++) - I915_WRITE(DEW + (i * 4), 0); + I915_WRITE(DEW(i), 0); /* Program P-state weights to account for frequency power adjustment */ for (i = 0; i < 16; i++) { - u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4)); + u32 pxvidfreq = I915_READ(PXVFREQ(i)); unsigned long freq = intel_pxfreq(pxvidfreq); unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; @@ -5686,7 +5978,7 @@ static void intel_init_emon(struct drm_device *dev) for (i = 0; i < 4; i++) { u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) | (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]); - I915_WRITE(PXW + (i * 4), val); + I915_WRITE(PXW(i), val); } /* Adjust magic regs to magic values (more experimental results) */ @@ -5702,7 +5994,7 @@ static void intel_init_emon(struct drm_device *dev) I915_WRITE(EG7, 0); for (i = 0; i < 8; i++) - I915_WRITE(PXWL + (i * 4), 0); + I915_WRITE(PXWL(i), 0); /* Enable PMON + select events */ I915_WRITE(ECR, 0x80000019); @@ -5802,7 +6094,8 @@ static void intel_gen6_powersave_work(struct work_struct *work) } else if (INTEL_INFO(dev)->gen >= 9) { gen9_enable_rc6(dev); gen9_enable_rps(dev); - __gen6_update_ring_freq(dev); + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) + __gen6_update_ring_freq(dev); } else if (IS_BROADWELL(dev)) { gen8_enable_rps(dev); __gen6_update_ring_freq(dev); @@ -6155,14 +6448,14 @@ static void lpt_init_clock_gating(struct drm_device *dev) * TODO: this bit should only be enabled when really needed, then * disabled when not needed anymore in order to save power. */ - if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) + if (HAS_PCH_LPT_LP(dev)) I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) | PCH_LP_PARTITION_LEVEL_DISABLE); /* WADPOClockGatingDisable:hsw */ - I915_WRITE(_TRANSA_CHICKEN1, - I915_READ(_TRANSA_CHICKEN1) | + I915_WRITE(TRANS_CHICKEN1(PIPE_A), + I915_READ(TRANS_CHICKEN1(PIPE_A)) | TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } @@ -6170,7 +6463,7 @@ static void lpt_suspend_hw(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { + if (HAS_PCH_LPT_LP(dev)) { uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; @@ -6656,11 +6949,7 @@ void intel_init_pm(struct drm_device *dev) if (IS_BROXTON(dev)) dev_priv->display.init_clock_gating = bxt_init_clock_gating; - else if (IS_SKYLAKE(dev)) - dev_priv->display.init_clock_gating = - skl_init_clock_gating; dev_priv->display.update_wm = skl_update_wm; - dev_priv->display.update_sprite_wm = skl_update_sprite_wm; } else if (HAS_PCH_SPLIT(dev)) { ilk_setup_wm_latency(dev); @@ -6669,7 +6958,7 @@ void intel_init_pm(struct drm_device *dev) (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { dev_priv->display.update_wm = ilk_update_wm; - dev_priv->display.update_sprite_wm = ilk_update_sprite_wm; + dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm; } else { DRM_DEBUG_KMS("Failed to read display plane latency. " "Disable CxSR\n"); @@ -6686,13 +6975,15 @@ void intel_init_pm(struct drm_device *dev) else if (INTEL_INFO(dev)->gen == 8) dev_priv->display.init_clock_gating = broadwell_init_clock_gating; } else if (IS_CHERRYVIEW(dev)) { - dev_priv->display.update_wm = valleyview_update_wm; - dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; + vlv_setup_wm_latency(dev); + + dev_priv->display.update_wm = vlv_update_wm; dev_priv->display.init_clock_gating = cherryview_init_clock_gating; } else if (IS_VALLEYVIEW(dev)) { - dev_priv->display.update_wm = valleyview_update_wm; - dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm; + vlv_setup_wm_latency(dev); + + dev_priv->display.update_wm = vlv_update_wm; dev_priv->display.init_clock_gating = valleyview_init_clock_gating; } else if (IS_PINEVIEW(dev)) { @@ -6809,7 +7100,7 @@ static int vlv_gpu_freq_div(unsigned int czclk_freq) static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) { - int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4); + int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); div = vlv_gpu_freq_div(czclk_freq); if (div < 0) @@ -6820,7 +7111,7 @@ static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) { - int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4); + int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); mul = vlv_gpu_freq_div(czclk_freq); if (mul < 0) @@ -6831,7 +7122,7 @@ static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) { - int div, czclk_freq = dev_priv->rps.cz_freq; + int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); div = vlv_gpu_freq_div(czclk_freq) / 2; if (div < 0) @@ -6842,7 +7133,7 @@ static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) { - int mul, czclk_freq = dev_priv->rps.cz_freq; + int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); mul = vlv_gpu_freq_div(czclk_freq) / 2; if (mul < 0) @@ -6855,7 +7146,8 @@ static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) { if (IS_GEN9(dev_priv->dev)) - return (val * GT_FREQUENCY_MULTIPLIER) / GEN9_FREQ_SCALER; + return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, + GEN9_FREQ_SCALER); else if (IS_CHERRYVIEW(dev_priv->dev)) return chv_gpu_freq(dev_priv, val); else if (IS_VALLEYVIEW(dev_priv->dev)) @@ -6867,13 +7159,14 @@ int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) { if (IS_GEN9(dev_priv->dev)) - return (val * GEN9_FREQ_SCALER) / GT_FREQUENCY_MULTIPLIER; + return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, + GT_FREQUENCY_MULTIPLIER); else if (IS_CHERRYVIEW(dev_priv->dev)) return chv_freq_opcode(dev_priv, val); else if (IS_VALLEYVIEW(dev_priv->dev)) return byt_freq_opcode(dev_priv, val); else - return val / GT_FREQUENCY_MULTIPLIER; + return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); } struct request_boost { diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c index 5ee0fa57ed19..b6609e648f75 100644 --- a/drivers/gpu/drm/i915/intel_psr.c +++ b/drivers/gpu/drm/i915/intel_psr.c @@ -73,14 +73,14 @@ static bool vlv_is_psr_active_on_pipe(struct drm_device *dev, int pipe) } static void intel_psr_write_vsc(struct intel_dp *intel_dp, - struct edp_vsc_psr *vsc_psr) + const struct edp_vsc_psr *vsc_psr) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc); - u32 ctl_reg = HSW_TVIDEO_DIP_CTL(crtc->config->cpu_transcoder); - u32 data_reg = HSW_TVIDEO_DIP_VSC_DATA(crtc->config->cpu_transcoder); + enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; + i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder); uint32_t *data = (uint32_t *) vsc_psr; unsigned int i; @@ -90,12 +90,14 @@ static void intel_psr_write_vsc(struct intel_dp *intel_dp, I915_WRITE(ctl_reg, 0); POSTING_READ(ctl_reg); - for (i = 0; i < VIDEO_DIP_VSC_DATA_SIZE; i += 4) { - if (i < sizeof(struct edp_vsc_psr)) - I915_WRITE(data_reg + i, *data++); - else - I915_WRITE(data_reg + i, 0); + for (i = 0; i < sizeof(*vsc_psr); i += 4) { + I915_WRITE(HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder, + i >> 2), *data); + data++; } + for (; i < VIDEO_DIP_VSC_DATA_SIZE; i += 4) + I915_WRITE(HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder, + i >> 2), 0); I915_WRITE(ctl_reg, VIDEO_DIP_ENABLE_VSC_HSW); POSTING_READ(ctl_reg); @@ -149,13 +151,31 @@ static void vlv_psr_enable_sink(struct intel_dp *intel_dp) DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE); } +static i915_reg_t psr_aux_ctl_reg(struct drm_i915_private *dev_priv, + enum port port) +{ + if (INTEL_INFO(dev_priv)->gen >= 9) + return DP_AUX_CH_CTL(port); + else + return EDP_PSR_AUX_CTL; +} + +static i915_reg_t psr_aux_data_reg(struct drm_i915_private *dev_priv, + enum port port, int index) +{ + if (INTEL_INFO(dev_priv)->gen >= 9) + return DP_AUX_CH_DATA(port, index); + else + return EDP_PSR_AUX_DATA(index); +} + static void hsw_psr_enable_sink(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; uint32_t aux_clock_divider; - uint32_t aux_data_reg, aux_ctl_reg; + i915_reg_t aux_ctl_reg; int precharge = 0x3; static const uint8_t aux_msg[] = { [0] = DP_AUX_NATIVE_WRITE << 4, @@ -164,29 +184,24 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp) [3] = 1 - 1, [4] = DP_SET_POWER_D0, }; + enum port port = dig_port->port; int i; BUILD_BUG_ON(sizeof(aux_msg) > 20); aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0); - drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, - DP_PSR_ENABLE & ~DP_PSR_MAIN_LINK_ACTIVE); - /* Enable AUX frame sync at sink */ if (dev_priv->psr.aux_frame_sync) drm_dp_dpcd_writeb(&intel_dp->aux, DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF, DP_AUX_FRAME_SYNC_ENABLE); - aux_data_reg = (INTEL_INFO(dev)->gen >= 9) ? - DPA_AUX_CH_DATA1 : EDP_PSR_AUX_DATA1(dev); - aux_ctl_reg = (INTEL_INFO(dev)->gen >= 9) ? - DPA_AUX_CH_CTL : EDP_PSR_AUX_CTL(dev); + aux_ctl_reg = psr_aux_ctl_reg(dev_priv, port); /* Setup AUX registers */ for (i = 0; i < sizeof(aux_msg); i += 4) - I915_WRITE(aux_data_reg + i, + I915_WRITE(psr_aux_data_reg(dev_priv, port, i >> 2), intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i)); if (INTEL_INFO(dev)->gen >= 9) { @@ -254,24 +269,22 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp) uint32_t max_sleep_time = 0x1f; /* Lately it was identified that depending on panel idle frame count * calculated at HW can be off by 1. So let's use what came - * from VBT + 1 and at minimum 2 to be on the safe side. + * from VBT + 1. + * There are also other cases where panel demands at least 4 + * but VBT is not being set. To cover these 2 cases lets use + * at least 5 when VBT isn't set to be on the safest side. */ uint32_t idle_frames = dev_priv->vbt.psr.idle_frames ? - dev_priv->vbt.psr.idle_frames + 1 : 2; + dev_priv->vbt.psr.idle_frames + 1 : 5; uint32_t val = 0x0; const uint32_t link_entry_time = EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES; if (intel_dp->psr_dpcd[1] & DP_PSR_NO_TRAIN_ON_EXIT) { - /* It doesn't mean we shouldn't send TPS patters, so let's - send the minimal TP1 possible and skip TP2. */ - val |= EDP_PSR_TP1_TIME_100us; - val |= EDP_PSR_TP2_TP3_TIME_0us; - val |= EDP_PSR_SKIP_AUX_EXIT; /* Sink should be able to train with the 5 or 6 idle patterns */ idle_frames += 4; } - I915_WRITE(EDP_PSR_CTL(dev), val | + I915_WRITE(EDP_PSR_CTL, val | (IS_BROADWELL(dev) ? 0 : link_entry_time) | max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT | idle_frames << EDP_PSR_IDLE_FRAME_SHIFT | @@ -335,7 +348,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp) struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - WARN_ON(I915_READ(EDP_PSR_CTL(dev)) & EDP_PSR_ENABLE); + WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE); WARN_ON(dev_priv->psr.active); lockdep_assert_held(&dev_priv->psr.lock); @@ -398,8 +411,13 @@ void intel_psr_enable(struct intel_dp *intel_dp) skl_psr_setup_su_vsc(intel_dp); } - /* Avoid continuous PSR exit by masking memup and hpd */ - I915_WRITE(EDP_PSR_DEBUG_CTL(dev), EDP_PSR_DEBUG_MASK_MEMUP | + /* + * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD. + * Also mask LPSP to avoid dependency on other drivers that + * might block runtime_pm besides preventing other hw tracking + * issues now we can rely on frontbuffer tracking. + */ + I915_WRITE(EDP_PSR_DEBUG_CTL, EDP_PSR_DEBUG_MASK_MEMUP | EDP_PSR_DEBUG_MASK_HPD | EDP_PSR_DEBUG_MASK_LPSP); /* Enable PSR on the panel */ @@ -422,6 +440,19 @@ void intel_psr_enable(struct intel_dp *intel_dp) vlv_psr_enable_source(intel_dp); } + /* + * FIXME: Activation should happen immediately since this function + * is just called after pipe is fully trained and enabled. + * However on every platform we face issues when first activation + * follows a modeset so quickly. + * - On VLV/CHV we get bank screen on first activation + * - On HSW/BDW we get a recoverable frozen screen until next + * exit-activate sequence. + */ + if (INTEL_INFO(dev)->gen < 9) + schedule_delayed_work(&dev_priv->psr.work, + msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5)); + dev_priv->psr.enabled = intel_dp; unlock: mutex_unlock(&dev_priv->psr.lock); @@ -461,17 +492,17 @@ static void hsw_psr_disable(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dev->dev_private; if (dev_priv->psr.active) { - I915_WRITE(EDP_PSR_CTL(dev), - I915_READ(EDP_PSR_CTL(dev)) & ~EDP_PSR_ENABLE); + I915_WRITE(EDP_PSR_CTL, + I915_READ(EDP_PSR_CTL) & ~EDP_PSR_ENABLE); /* Wait till PSR is idle */ - if (_wait_for((I915_READ(EDP_PSR_STATUS_CTL(dev)) & + if (_wait_for((I915_READ(EDP_PSR_STATUS_CTL) & EDP_PSR_STATUS_STATE_MASK) == 0, 2000, 10)) DRM_ERROR("Timed out waiting for PSR Idle State\n"); dev_priv->psr.active = false; } else { - WARN_ON(I915_READ(EDP_PSR_CTL(dev)) & EDP_PSR_ENABLE); + WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE); } } @@ -493,11 +524,15 @@ void intel_psr_disable(struct intel_dp *intel_dp) return; } + /* Disable PSR on Source */ if (HAS_DDI(dev)) hsw_psr_disable(intel_dp); else vlv_psr_disable(intel_dp); + /* Disable PSR on Sink */ + drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0); + dev_priv->psr.enabled = NULL; mutex_unlock(&dev_priv->psr.lock); @@ -518,7 +553,7 @@ static void intel_psr_work(struct work_struct *work) * and be ready for re-enable. */ if (HAS_DDI(dev_priv->dev)) { - if (wait_for((I915_READ(EDP_PSR_STATUS_CTL(dev_priv->dev)) & + if (wait_for((I915_READ(EDP_PSR_STATUS_CTL) & EDP_PSR_STATUS_STATE_MASK) == 0, 50)) { DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n"); return; @@ -561,11 +596,11 @@ static void intel_psr_exit(struct drm_device *dev) return; if (HAS_DDI(dev)) { - val = I915_READ(EDP_PSR_CTL(dev)); + val = I915_READ(EDP_PSR_CTL); WARN_ON(!(val & EDP_PSR_ENABLE)); - I915_WRITE(EDP_PSR_CTL(dev), val & ~EDP_PSR_ENABLE); + I915_WRITE(EDP_PSR_CTL, val & ~EDP_PSR_ENABLE); } else { val = I915_READ(VLV_PSRCTL(pipe)); @@ -596,13 +631,15 @@ static void intel_psr_exit(struct drm_device *dev) /** * intel_psr_single_frame_update - Single Frame Update * @dev: DRM device + * @frontbuffer_bits: frontbuffer plane tracking bits * * Some platforms support a single frame update feature that is used to * send and update only one frame on Remote Frame Buffer. * So far it is only implemented for Valleyview and Cherryview because * hardware requires this to be done before a page flip. */ -void intel_psr_single_frame_update(struct drm_device *dev) +void intel_psr_single_frame_update(struct drm_device *dev, + unsigned frontbuffer_bits) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; @@ -624,14 +661,16 @@ void intel_psr_single_frame_update(struct drm_device *dev) crtc = dp_to_dig_port(dev_priv->psr.enabled)->base.base.crtc; pipe = to_intel_crtc(crtc)->pipe; - val = I915_READ(VLV_PSRCTL(pipe)); - /* - * We need to set this bit before writing registers for a flip. - * This bit will be self-clear when it gets to the PSR active state. - */ - I915_WRITE(VLV_PSRCTL(pipe), val | VLV_EDP_PSR_SINGLE_FRAME_UPDATE); + if (frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(pipe)) { + val = I915_READ(VLV_PSRCTL(pipe)); + /* + * We need to set this bit before writing registers for a flip. + * This bit will be self-clear when it gets to the PSR active state. + */ + I915_WRITE(VLV_PSRCTL(pipe), val | VLV_EDP_PSR_SINGLE_FRAME_UPDATE); + } mutex_unlock(&dev_priv->psr.lock); } @@ -648,7 +687,7 @@ void intel_psr_single_frame_update(struct drm_device *dev) * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits." */ void intel_psr_invalidate(struct drm_device *dev, - unsigned frontbuffer_bits) + unsigned frontbuffer_bits) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; @@ -663,11 +702,12 @@ void intel_psr_invalidate(struct drm_device *dev, crtc = dp_to_dig_port(dev_priv->psr.enabled)->base.base.crtc; pipe = to_intel_crtc(crtc)->pipe; - intel_psr_exit(dev); - frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); - dev_priv->psr.busy_frontbuffer_bits |= frontbuffer_bits; + + if (frontbuffer_bits) + intel_psr_exit(dev); + mutex_unlock(&dev_priv->psr.lock); } @@ -675,6 +715,7 @@ void intel_psr_invalidate(struct drm_device *dev, * intel_psr_flush - Flush PSR * @dev: DRM device * @frontbuffer_bits: frontbuffer plane tracking bits + * @origin: which operation caused the flush * * Since the hardware frontbuffer tracking has gaps we need to integrate * with the software frontbuffer tracking. This function gets called every @@ -684,7 +725,7 @@ void intel_psr_invalidate(struct drm_device *dev, * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits. */ void intel_psr_flush(struct drm_device *dev, - unsigned frontbuffer_bits) + unsigned frontbuffer_bits, enum fb_op_origin origin) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; @@ -698,30 +739,18 @@ void intel_psr_flush(struct drm_device *dev, crtc = dp_to_dig_port(dev_priv->psr.enabled)->base.base.crtc; pipe = to_intel_crtc(crtc)->pipe; - dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits; - /* - * On Haswell sprite plane updates don't result in a psr invalidating - * signal in the hardware. Which means we need to manually fake this in - * software for all flushes, not just when we've seen a preceding - * invalidation through frontbuffer rendering. - */ - if (IS_HASWELL(dev) && - (frontbuffer_bits & INTEL_FRONTBUFFER_SPRITE(pipe))) - intel_psr_exit(dev); + frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(pipe); + dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits; - /* - * On Valleyview and Cherryview we don't use hardware tracking so - * any plane updates or cursor moves don't result in a PSR - * invalidating. Which means we need to manually fake this in - * software for all flushes, not just when we've seen a preceding - * invalidation through frontbuffer rendering. */ - if (!HAS_DDI(dev)) + /* By definition flush = invalidate + flush */ + if (frontbuffer_bits) intel_psr_exit(dev); if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits) - schedule_delayed_work(&dev_priv->psr.work, - msecs_to_jiffies(100)); + if (!work_busy(&dev_priv->psr.work.work)) + schedule_delayed_work(&dev_priv->psr.work, + msecs_to_jiffies(100)); mutex_unlock(&dev_priv->psr.lock); } @@ -736,6 +765,9 @@ void intel_psr_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ? + HSW_EDP_PSR_BASE : BDW_EDP_PSR_BASE; + INIT_DELAYED_WORK(&dev_priv->psr.work, intel_psr_work); mutex_init(&dev_priv->psr.lock); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 3817a6f00d9e..57d78f264b53 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -81,7 +81,7 @@ bool intel_ring_stopped(struct intel_engine_cs *ring) return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring); } -void __intel_ring_advance(struct intel_engine_cs *ring) +static void __intel_ring_advance(struct intel_engine_cs *ring) { struct intel_ringbuffer *ringbuf = ring->buffer; ringbuf->tail &= ringbuf->size - 1; @@ -91,10 +91,11 @@ void __intel_ring_advance(struct intel_engine_cs *ring) } static int -gen2_render_ring_flush(struct intel_engine_cs *ring, +gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 invalidate_domains, u32 flush_domains) { + struct intel_engine_cs *ring = req->ring; u32 cmd; int ret; @@ -105,7 +106,7 @@ gen2_render_ring_flush(struct intel_engine_cs *ring, if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER) cmd |= MI_READ_FLUSH; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; @@ -117,10 +118,11 @@ gen2_render_ring_flush(struct intel_engine_cs *ring, } static int -gen4_render_ring_flush(struct intel_engine_cs *ring, +gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 invalidate_domains, u32 flush_domains) { + struct intel_engine_cs *ring = req->ring; struct drm_device *dev = ring->dev; u32 cmd; int ret; @@ -163,7 +165,7 @@ gen4_render_ring_flush(struct intel_engine_cs *ring, (IS_G4X(dev) || IS_GEN5(dev))) cmd |= MI_INVALIDATE_ISP; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; @@ -212,13 +214,13 @@ gen4_render_ring_flush(struct intel_engine_cs *ring, * really our business. That leaves only stall at scoreboard. */ static int -intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) +intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; - - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; @@ -231,7 +233,7 @@ intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; @@ -247,15 +249,16 @@ intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) } static int -gen6_render_ring_flush(struct intel_engine_cs *ring, - u32 invalidate_domains, u32 flush_domains) +gen6_render_ring_flush(struct drm_i915_gem_request *req, + u32 invalidate_domains, u32 flush_domains) { + struct intel_engine_cs *ring = req->ring; u32 flags = 0; u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; /* Force SNB workarounds for PIPE_CONTROL flushes */ - ret = intel_emit_post_sync_nonzero_flush(ring); + ret = intel_emit_post_sync_nonzero_flush(req); if (ret) return ret; @@ -285,7 +288,7 @@ gen6_render_ring_flush(struct intel_engine_cs *ring, flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; } - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -299,11 +302,12 @@ gen6_render_ring_flush(struct intel_engine_cs *ring, } static int -gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) +gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -318,9 +322,10 @@ gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) } static int -gen7_render_ring_flush(struct intel_engine_cs *ring, +gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 invalidate_domains, u32 flush_domains) { + struct intel_engine_cs *ring = req->ring; u32 flags = 0; u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; @@ -342,6 +347,7 @@ gen7_render_ring_flush(struct intel_engine_cs *ring, if (flush_domains) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; + flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (invalidate_domains) { flags |= PIPE_CONTROL_TLB_INVALIDATE; @@ -362,10 +368,10 @@ gen7_render_ring_flush(struct intel_engine_cs *ring, /* Workaround: we must issue a pipe_control with CS-stall bit * set before a pipe_control command that has the state cache * invalidate bit set. */ - gen7_render_ring_cs_stall_wa(ring); + gen7_render_ring_cs_stall_wa(req); } - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -379,12 +385,13 @@ gen7_render_ring_flush(struct intel_engine_cs *ring, } static int -gen8_emit_pipe_control(struct intel_engine_cs *ring, +gen8_emit_pipe_control(struct drm_i915_gem_request *req, u32 flags, u32 scratch_addr) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; @@ -400,11 +407,11 @@ gen8_emit_pipe_control(struct intel_engine_cs *ring, } static int -gen8_render_ring_flush(struct intel_engine_cs *ring, +gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 invalidate_domains, u32 flush_domains) { u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; + u32 scratch_addr = req->ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; flags |= PIPE_CONTROL_CS_STALL; @@ -412,6 +419,7 @@ gen8_render_ring_flush(struct intel_engine_cs *ring, if (flush_domains) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; + flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (invalidate_domains) { flags |= PIPE_CONTROL_TLB_INVALIDATE; @@ -424,7 +432,7 @@ gen8_render_ring_flush(struct intel_engine_cs *ring, flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */ - ret = gen8_emit_pipe_control(ring, + ret = gen8_emit_pipe_control(req, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD, 0); @@ -432,7 +440,7 @@ gen8_render_ring_flush(struct intel_engine_cs *ring, return ret; } - return gen8_emit_pipe_control(ring, flags, scratch_addr); + return gen8_emit_pipe_control(req, flags, scratch_addr); } static void ring_write_tail(struct intel_engine_cs *ring, @@ -473,7 +481,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = ring->dev->dev_private; - u32 mmio = 0; + i915_reg_t mmio; /* The ring status page addresses are no longer next to the rest of * the ring registers as of gen7. @@ -516,7 +524,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *ring) * invalidating the TLB? */ if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) { - u32 reg = RING_INSTPM(ring->mmio_base); + i915_reg_t reg = RING_INSTPM(ring->mmio_base); /* ring should be idle before issuing a sync flush*/ WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); @@ -703,29 +711,29 @@ err: return ret; } -static int intel_ring_workarounds_emit(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req) { int ret, i; + struct intel_engine_cs *ring = req->ring; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct i915_workarounds *w = &dev_priv->workarounds; - if (WARN_ON_ONCE(w->count == 0)) + if (w->count == 0) return 0; ring->gpu_caches_dirty = true; - ret = intel_ring_flush_all_caches(ring); + ret = intel_ring_flush_all_caches(req); if (ret) return ret; - ret = intel_ring_begin(ring, (w->count * 2 + 2)); + ret = intel_ring_begin(req, (w->count * 2 + 2)); if (ret) return ret; intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count)); for (i = 0; i < w->count; i++) { - intel_ring_emit(ring, w->reg[i].addr); + intel_ring_emit_reg(ring, w->reg[i].addr); intel_ring_emit(ring, w->reg[i].value); } intel_ring_emit(ring, MI_NOOP); @@ -733,7 +741,7 @@ static int intel_ring_workarounds_emit(struct intel_engine_cs *ring, intel_ring_advance(ring); ring->gpu_caches_dirty = true; - ret = intel_ring_flush_all_caches(ring); + ret = intel_ring_flush_all_caches(req); if (ret) return ret; @@ -742,16 +750,15 @@ static int intel_ring_workarounds_emit(struct intel_engine_cs *ring, return 0; } -static int intel_rcs_ctx_init(struct intel_engine_cs *ring, - struct intel_context *ctx) +static int intel_rcs_ctx_init(struct drm_i915_gem_request *req) { int ret; - ret = intel_ring_workarounds_emit(ring, ctx); + ret = intel_ring_workarounds_emit(req); if (ret != 0) return ret; - ret = i915_gem_render_state_init(ring); + ret = i915_gem_render_state_init(req); if (ret) DRM_ERROR("init render state: %d\n", ret); @@ -759,7 +766,8 @@ static int intel_rcs_ctx_init(struct intel_engine_cs *ring, } static int wa_add(struct drm_i915_private *dev_priv, - const u32 addr, const u32 mask, const u32 val) + i915_reg_t addr, + const u32 mask, const u32 val) { const u32 idx = dev_priv->workarounds.count; @@ -775,11 +783,11 @@ static int wa_add(struct drm_i915_private *dev_priv, return 0; } -#define WA_REG(addr, mask, val) { \ +#define WA_REG(addr, mask, val) do { \ const int r = wa_add(dev_priv, (addr), (mask), (val)); \ if (r) \ return r; \ - } + } while (0) #define WA_SET_BIT_MASKED(addr, mask) \ WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask)) @@ -795,37 +803,29 @@ static int wa_add(struct drm_i915_private *dev_priv, #define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val) -static int bdw_init_workarounds(struct intel_engine_cs *ring) +static int gen8_init_workarounds(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - /* WaDisablePartialInstShootdown:bdw */ - /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ - WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, - PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE | - STALL_DOP_GATING_DISABLE); + WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); - /* WaDisableDopClockGating:bdw */ - WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, - DOP_CLOCK_GATING_DISABLE); + /* WaDisableAsyncFlipPerfMode:bdw,chv */ + WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE); - WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, - GEN8_SAMPLER_POWER_BYPASS_DIS); + /* WaDisablePartialInstShootdown:bdw,chv */ + WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, + PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); /* Use Force Non-Coherent whenever executing a 3D context. This is a * workaround for for a possible hang in the unlikely event a TLB * invalidation occurs during a PSD flush. */ + /* WaForceEnableNonCoherent:bdw,chv */ + /* WaHdcDisableFetchWhenMasked:bdw,chv */ WA_SET_BIT_MASKED(HDC_CHICKEN0, - /* WaForceEnableNonCoherent:bdw */ - HDC_FORCE_NON_COHERENT | - /* WaForceContextSaveRestoreNonCoherent:bdw */ - HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | - /* WaHdcDisableFetchWhenMasked:bdw */ HDC_DONOT_FETCH_MEM_WHEN_MASKED | - /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ - (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); + HDC_FORCE_NON_COHERENT); /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0: * "The Hierarchical Z RAW Stall Optimization allows non-overlapping @@ -833,13 +833,12 @@ static int bdw_init_workarounds(struct intel_engine_cs *ring) * stalling waiting for the earlier ones to write to Hierarchical Z * buffer." * - * This optimization is off by default for Broadwell; turn it on. + * This optimization is off by default for BDW and CHV; turn it on. */ WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); - /* Wa4x4STCOptimizationDisable:bdw */ - WA_SET_BIT_MASKED(CACHE_MODE_1, - GEN8_4x4_STC_OPTIMIZATION_DISABLE); + /* Wa4x4STCOptimizationDisable:bdw,chv */ + WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE); /* * BSpec recommends 8x4 when MSAA is used, @@ -856,51 +855,51 @@ static int bdw_init_workarounds(struct intel_engine_cs *ring) return 0; } -static int chv_init_workarounds(struct intel_engine_cs *ring) +static int bdw_init_workarounds(struct intel_engine_cs *ring) { + int ret; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - /* WaDisablePartialInstShootdown:chv */ - /* WaDisableThreadStallDopClockGating:chv */ - WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, - PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE | - STALL_DOP_GATING_DISABLE); + ret = gen8_init_workarounds(ring); + if (ret) + return ret; + + /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ + WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); + + /* WaDisableDopClockGating:bdw */ + WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, + DOP_CLOCK_GATING_DISABLE); + + WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, + GEN8_SAMPLER_POWER_BYPASS_DIS); - /* Use Force Non-Coherent whenever executing a 3D context. This is a - * workaround for a possible hang in the unlikely event a TLB - * invalidation occurs during a PSD flush. - */ - /* WaForceEnableNonCoherent:chv */ - /* WaHdcDisableFetchWhenMasked:chv */ WA_SET_BIT_MASKED(HDC_CHICKEN0, - HDC_FORCE_NON_COHERENT | - HDC_DONOT_FETCH_MEM_WHEN_MASKED); + /* WaForceContextSaveRestoreNonCoherent:bdw */ + HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | + /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ + (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); - /* According to the CACHE_MODE_0 default value documentation, some - * CHV platforms disable this optimization by default. Turn it on. - */ - WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); + return 0; +} - /* Wa4x4STCOptimizationDisable:chv */ - WA_SET_BIT_MASKED(CACHE_MODE_1, - GEN8_4x4_STC_OPTIMIZATION_DISABLE); +static int chv_init_workarounds(struct intel_engine_cs *ring) +{ + int ret; + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + ret = gen8_init_workarounds(ring); + if (ret) + return ret; + + /* WaDisableThreadStallDopClockGating:chv */ + WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); /* Improve HiZ throughput on CHV. */ WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X); - /* - * BSpec recommends 8x4 when MSAA is used, - * however in practice 16x4 seems fastest. - * - * Note that PS/WM thread counts depend on the WIZ hashing - * disable bit, which we don't touch here, but it's good - * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). - */ - WA_SET_FIELD_MASKED(GEN7_GT_MODE, - GEN6_WIZ_HASHING_MASK, - GEN6_WIZ_HASHING_16x4); - return 0; } @@ -910,6 +909,14 @@ static int gen9_init_workarounds(struct intel_engine_cs *ring) struct drm_i915_private *dev_priv = dev->dev_private; uint32_t tmp; + /* WaEnableLbsSlaRetryTimerDecrement:skl */ + I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) | + GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); + + /* WaDisableKillLogic:bxt,skl */ + I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | + ECOCHK_DIS_TLB); + /* WaDisablePartialInstShootdown:skl,bxt */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); @@ -918,53 +925,59 @@ static int gen9_init_workarounds(struct intel_engine_cs *ring) WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC); - if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) == SKL_REVID_A0 || - INTEL_REVID(dev) == SKL_REVID_B0)) || - (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) { - /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */ + /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, GEN9_DG_MIRROR_FIX_ENABLE); - } - if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_B0) || - (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) { - /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ + /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1, GEN9_RHWO_OPTIMIZATION_DISABLE); - WA_SET_BIT_MASKED(GEN9_SLICE_COMMON_ECO_CHICKEN0, - DISABLE_PIXEL_MASK_CAMMING); + /* + * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set + * but we do that in per ctx batchbuffer as there is an issue + * with this register not getting restored on ctx restore + */ } - if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) >= SKL_REVID_C0) || - IS_BROXTON(dev)) { - /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */ + /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */ + if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER) || IS_BROXTON(dev)) WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7, GEN9_ENABLE_YV12_BUGFIX); - } /* Wa4x4STCOptimizationDisable:skl,bxt */ - WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE); - /* WaDisablePartialResolveInVc:skl,bxt */ - WA_SET_BIT_MASKED(CACHE_MODE_1, GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE); + WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE | + GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE)); /* WaCcsTlbPrefetchDisable:skl,bxt */ WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, GEN9_CCS_TLB_PREFETCH_ENABLE); /* WaDisableMaskBasedCammingInRCC:skl,bxt */ - if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_C0) || - (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)) + if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) || + IS_BXT_REVID(dev, 0, BXT_REVID_A1)) WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0, PIXEL_MASK_CAMMING_DISABLE); /* WaForceContextSaveRestoreNonCoherent:skl,bxt */ tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT; - if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) == SKL_REVID_F0) || - (IS_BROXTON(dev) && INTEL_REVID(dev) >= BXT_REVID_B0)) + if (IS_SKL_REVID(dev, SKL_REVID_F0, SKL_REVID_F0) || + IS_BXT_REVID(dev, BXT_REVID_B0, REVID_FOREVER)) tmp |= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE; WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp); + /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */ + if (IS_SKYLAKE(dev) || IS_BXT_REVID(dev, 0, BXT_REVID_B0)) + WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, + GEN8_SAMPLER_POWER_BYPASS_DIS); + + /* WaDisableSTUnitPowerOptimization:skl,bxt */ + WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); + return 0; } @@ -1010,27 +1023,46 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *ring) return 0; } - static int skl_init_workarounds(struct intel_engine_cs *ring) { + int ret; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - gen9_init_workarounds(ring); + ret = gen9_init_workarounds(ring); + if (ret) + return ret; + + if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) { + /* WaDisableHDCInvalidation:skl */ + I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | + BDW_DISABLE_HDC_INVALIDATION); + + /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */ + I915_WRITE(FF_SLICE_CS_CHICKEN2, + _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE)); + } + + /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes + * involving this register should also be added to WA batch as required. + */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_E0)) + /* WaDisableLSQCROPERFforOCL:skl */ + I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | + GEN8_LQSC_RO_PERF_DIS); + + /* WaEnableGapsTsvCreditFix:skl */ + if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) { + I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) | + GEN9_GAPS_TSV_CREDIT_DISABLE)); + } /* WaDisablePowerCompilerClockGating:skl */ - if (INTEL_REVID(dev) == SKL_REVID_B0) + if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0)) WA_SET_BIT_MASKED(HIZ_CHICKEN, BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE); - if (INTEL_REVID(dev) == SKL_REVID_C0 || - INTEL_REVID(dev) == SKL_REVID_D0) - /* WaBarrierPerformanceFixDisable:skl */ - WA_SET_BIT_MASKED(HDC_CHICKEN0, - HDC_FENCE_DEST_SLM_DISABLE | - HDC_BARRIER_PERFORMANCE_DISABLE); - - if (INTEL_REVID(dev) <= SKL_REVID_D0) { + if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) { /* *Use Force Non-Coherent whenever executing a 3D context. This * is a workaround for a possible hang in the unlikely event @@ -1041,22 +1073,48 @@ static int skl_init_workarounds(struct intel_engine_cs *ring) HDC_FORCE_NON_COHERENT); } + /* WaBarrierPerformanceFixDisable:skl */ + if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0)) + WA_SET_BIT_MASKED(HDC_CHICKEN0, + HDC_FENCE_DEST_SLM_DISABLE | + HDC_BARRIER_PERFORMANCE_DISABLE); + + /* WaDisableSbeCacheDispatchPortSharing:skl */ + if (IS_SKL_REVID(dev, 0, SKL_REVID_F0)) + WA_SET_BIT_MASKED( + GEN7_HALF_SLICE_CHICKEN1, + GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); + return skl_tune_iz_hashing(ring); } static int bxt_init_workarounds(struct intel_engine_cs *ring) { + int ret; struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - gen9_init_workarounds(ring); + ret = gen9_init_workarounds(ring); + if (ret) + return ret; + + /* WaStoreMultiplePTEenable:bxt */ + /* This is a requirement according to Hardware specification */ + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF); + + /* WaSetClckGatingDisableMedia:bxt */ + if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & + ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE)); + } /* WaDisableThreadStallDopClockGating:bxt */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); /* WaDisableSbeCacheDispatchPortSharing:bxt */ - if (INTEL_REVID(dev) <= BXT_REVID_B0) { + if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) { WA_SET_BIT_MASKED( GEN7_HALF_SLICE_CHICKEN1, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); @@ -1105,9 +1163,9 @@ static int init_render_ring(struct intel_engine_cs *ring) * to use MI_WAIT_FOR_EVENT within the CS. It should already be * programmed to '1' on all products. * - * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv + * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv */ - if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 9) + if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); /* Required for the hardware to program scanline values for waiting */ @@ -1132,7 +1190,7 @@ static int init_render_ring(struct intel_engine_cs *ring) _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); } - if (INTEL_INFO(dev)->gen >= 6) + if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); if (HAS_L3_DPF(dev)) @@ -1155,10 +1213,11 @@ static void render_ring_cleanup(struct intel_engine_cs *ring) intel_fini_pipe_control(ring); } -static int gen8_rcs_signal(struct intel_engine_cs *signaller, +static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req, unsigned int num_dwords) { #define MBOX_UPDATE_DWORDS 8 + struct intel_engine_cs *signaller = signaller_req->ring; struct drm_device *dev = signaller->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *waiter; @@ -1168,7 +1227,7 @@ static int gen8_rcs_signal(struct intel_engine_cs *signaller, num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; #undef MBOX_UPDATE_DWORDS - ret = intel_ring_begin(signaller, num_dwords); + ret = intel_ring_begin(signaller_req, num_dwords); if (ret) return ret; @@ -1178,8 +1237,7 @@ static int gen8_rcs_signal(struct intel_engine_cs *signaller, if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - seqno = i915_gem_request_get_seqno( - signaller->outstanding_lazy_request); + seqno = i915_gem_request_get_seqno(signaller_req); intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6)); intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE | @@ -1196,10 +1254,11 @@ static int gen8_rcs_signal(struct intel_engine_cs *signaller, return 0; } -static int gen8_xcs_signal(struct intel_engine_cs *signaller, +static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req, unsigned int num_dwords) { #define MBOX_UPDATE_DWORDS 6 + struct intel_engine_cs *signaller = signaller_req->ring; struct drm_device *dev = signaller->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *waiter; @@ -1209,7 +1268,7 @@ static int gen8_xcs_signal(struct intel_engine_cs *signaller, num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; #undef MBOX_UPDATE_DWORDS - ret = intel_ring_begin(signaller, num_dwords); + ret = intel_ring_begin(signaller_req, num_dwords); if (ret) return ret; @@ -1219,8 +1278,7 @@ static int gen8_xcs_signal(struct intel_engine_cs *signaller, if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - seqno = i915_gem_request_get_seqno( - signaller->outstanding_lazy_request); + seqno = i915_gem_request_get_seqno(signaller_req); intel_ring_emit(signaller, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); intel_ring_emit(signaller, lower_32_bits(gtt_offset) | @@ -1235,9 +1293,10 @@ static int gen8_xcs_signal(struct intel_engine_cs *signaller, return 0; } -static int gen6_signal(struct intel_engine_cs *signaller, +static int gen6_signal(struct drm_i915_gem_request *signaller_req, unsigned int num_dwords) { + struct intel_engine_cs *signaller = signaller_req->ring; struct drm_device *dev = signaller->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *useless; @@ -1248,17 +1307,18 @@ static int gen6_signal(struct intel_engine_cs *signaller, num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2); #undef MBOX_UPDATE_DWORDS - ret = intel_ring_begin(signaller, num_dwords); + ret = intel_ring_begin(signaller_req, num_dwords); if (ret) return ret; for_each_ring(useless, dev_priv, i) { - u32 mbox_reg = signaller->semaphore.mbox.signal[i]; - if (mbox_reg != GEN6_NOSYNC) { - u32 seqno = i915_gem_request_get_seqno( - signaller->outstanding_lazy_request); + i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[i]; + + if (i915_mmio_reg_valid(mbox_reg)) { + u32 seqno = i915_gem_request_get_seqno(signaller_req); + intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(signaller, mbox_reg); + intel_ring_emit_reg(signaller, mbox_reg); intel_ring_emit(signaller, seqno); } } @@ -1272,30 +1332,29 @@ static int gen6_signal(struct intel_engine_cs *signaller, /** * gen6_add_request - Update the semaphore mailbox registers - * - * @ring - ring that is adding a request - * @seqno - return seqno stuck into the ring + * + * @request - request to write to the ring * * Update the mailbox registers in the *other* rings with the current seqno. * This acts like a signal in the canonical semaphore. */ static int -gen6_add_request(struct intel_engine_cs *ring) +gen6_add_request(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int ret; if (ring->semaphore.signal) - ret = ring->semaphore.signal(ring, 4); + ret = ring->semaphore.signal(req, 4); else - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, - i915_gem_request_get_seqno(ring->outstanding_lazy_request)); + intel_ring_emit(ring, i915_gem_request_get_seqno(req)); intel_ring_emit(ring, MI_USER_INTERRUPT); __intel_ring_advance(ring); @@ -1318,14 +1377,15 @@ static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, */ static int -gen8_ring_sync(struct intel_engine_cs *waiter, +gen8_ring_sync(struct drm_i915_gem_request *waiter_req, struct intel_engine_cs *signaller, u32 seqno) { + struct intel_engine_cs *waiter = waiter_req->ring; struct drm_i915_private *dev_priv = waiter->dev->dev_private; int ret; - ret = intel_ring_begin(waiter, 4); + ret = intel_ring_begin(waiter_req, 4); if (ret) return ret; @@ -1343,10 +1403,11 @@ gen8_ring_sync(struct intel_engine_cs *waiter, } static int -gen6_ring_sync(struct intel_engine_cs *waiter, +gen6_ring_sync(struct drm_i915_gem_request *waiter_req, struct intel_engine_cs *signaller, u32 seqno) { + struct intel_engine_cs *waiter = waiter_req->ring; u32 dw1 = MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER; @@ -1361,7 +1422,7 @@ gen6_ring_sync(struct intel_engine_cs *waiter, WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID); - ret = intel_ring_begin(waiter, 4); + ret = intel_ring_begin(waiter_req, 4); if (ret) return ret; @@ -1392,8 +1453,9 @@ do { \ } while (0) static int -pc_render_add_request(struct intel_engine_cs *ring) +pc_render_add_request(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; @@ -1405,7 +1467,7 @@ pc_render_add_request(struct intel_engine_cs *ring) * incoherence by flushing the 6 PIPE_NOTIFY buffers out to * memory before requesting an interrupt. */ - ret = intel_ring_begin(ring, 32); + ret = intel_ring_begin(req, 32); if (ret) return ret; @@ -1413,8 +1475,7 @@ pc_render_add_request(struct intel_engine_cs *ring) PIPE_CONTROL_WRITE_FLUSH | PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(ring, - i915_gem_request_get_seqno(ring->outstanding_lazy_request)); + intel_ring_emit(ring, i915_gem_request_get_seqno(req)); intel_ring_emit(ring, 0); PIPE_CONTROL_FLUSH(ring, scratch_addr); scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */ @@ -1433,8 +1494,7 @@ pc_render_add_request(struct intel_engine_cs *ring) PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | PIPE_CONTROL_NOTIFY); intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(ring, - i915_gem_request_get_seqno(ring->outstanding_lazy_request)); + intel_ring_emit(ring, i915_gem_request_get_seqno(req)); intel_ring_emit(ring, 0); __intel_ring_advance(ring); @@ -1585,13 +1645,14 @@ i8xx_ring_put_irq(struct intel_engine_cs *ring) } static int -bsd_ring_flush(struct intel_engine_cs *ring, +bsd_ring_flush(struct drm_i915_gem_request *req, u32 invalidate_domains, u32 flush_domains) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; @@ -1602,18 +1663,18 @@ bsd_ring_flush(struct intel_engine_cs *ring, } static int -i9xx_add_request(struct intel_engine_cs *ring) +i9xx_add_request(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, - i915_gem_request_get_seqno(ring->outstanding_lazy_request)); + intel_ring_emit(ring, i915_gem_request_get_seqno(req)); intel_ring_emit(ring, MI_USER_INTERRUPT); __intel_ring_advance(ring); @@ -1745,13 +1806,14 @@ gen8_ring_put_irq(struct intel_engine_cs *ring) } static int -i965_dispatch_execbuffer(struct intel_engine_cs *ring, +i965_dispatch_execbuffer(struct drm_i915_gem_request *req, u64 offset, u32 length, unsigned dispatch_flags) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; @@ -1771,14 +1833,15 @@ i965_dispatch_execbuffer(struct intel_engine_cs *ring, #define I830_TLB_ENTRIES (2) #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) static int -i830_dispatch_execbuffer(struct intel_engine_cs *ring, +i830_dispatch_execbuffer(struct drm_i915_gem_request *req, u64 offset, u32 len, unsigned dispatch_flags) { + struct intel_engine_cs *ring = req->ring; u32 cs_offset = ring->scratch.gtt_offset; int ret; - ret = intel_ring_begin(ring, 6); + ret = intel_ring_begin(req, 6); if (ret) return ret; @@ -1795,7 +1858,7 @@ i830_dispatch_execbuffer(struct intel_engine_cs *ring, if (len > I830_BATCH_LIMIT) return -ENOSPC; - ret = intel_ring_begin(ring, 6 + 2); + ret = intel_ring_begin(req, 6 + 2); if (ret) return ret; @@ -1818,7 +1881,7 @@ i830_dispatch_execbuffer(struct intel_engine_cs *ring, offset = cs_offset; } - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -1833,13 +1896,14 @@ i830_dispatch_execbuffer(struct intel_engine_cs *ring, } static int -i915_dispatch_execbuffer(struct intel_engine_cs *ring, +i915_dispatch_execbuffer(struct drm_i915_gem_request *req, u64 offset, u32 len, unsigned dispatch_flags) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; @@ -1935,11 +1999,35 @@ static int init_phys_status_page(struct intel_engine_cs *ring) void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf) { - iounmap(ringbuf->virtual_start); + if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen) + vunmap(ringbuf->virtual_start); + else + iounmap(ringbuf->virtual_start); ringbuf->virtual_start = NULL; i915_gem_object_ggtt_unpin(ringbuf->obj); } +static u32 *vmap_obj(struct drm_i915_gem_object *obj) +{ + struct sg_page_iter sg_iter; + struct page **pages; + void *addr; + int i; + + pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages)); + if (pages == NULL) + return NULL; + + i = 0; + for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) + pages[i++] = sg_page_iter_page(&sg_iter); + + addr = vmap(pages, i, 0, PAGE_KERNEL); + drm_free_large(pages); + + return addr; +} + int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, struct intel_ringbuffer *ringbuf) { @@ -1947,34 +2035,52 @@ int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, struct drm_i915_gem_object *obj = ringbuf->obj; int ret; - ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE); - if (ret) - return ret; + if (HAS_LLC(dev_priv) && !obj->stolen) { + ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, 0); + if (ret) + return ret; - ret = i915_gem_object_set_to_gtt_domain(obj, true); - if (ret) { - i915_gem_object_ggtt_unpin(obj); - return ret; - } + ret = i915_gem_object_set_to_cpu_domain(obj, true); + if (ret) { + i915_gem_object_ggtt_unpin(obj); + return ret; + } - ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base + - i915_gem_obj_ggtt_offset(obj), ringbuf->size); - if (ringbuf->virtual_start == NULL) { - i915_gem_object_ggtt_unpin(obj); - return -EINVAL; + ringbuf->virtual_start = vmap_obj(obj); + if (ringbuf->virtual_start == NULL) { + i915_gem_object_ggtt_unpin(obj); + return -ENOMEM; + } + } else { + ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE); + if (ret) + return ret; + + ret = i915_gem_object_set_to_gtt_domain(obj, true); + if (ret) { + i915_gem_object_ggtt_unpin(obj); + return ret; + } + + ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base + + i915_gem_obj_ggtt_offset(obj), ringbuf->size); + if (ringbuf->virtual_start == NULL) { + i915_gem_object_ggtt_unpin(obj); + return -EINVAL; + } } return 0; } -void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf) +static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf) { drm_gem_object_unreference(&ringbuf->obj->base); ringbuf->obj = NULL; } -int intel_alloc_ringbuffer_obj(struct drm_device *dev, - struct intel_ringbuffer *ringbuf) +static int intel_alloc_ringbuffer_obj(struct drm_device *dev, + struct intel_ringbuffer *ringbuf) { struct drm_i915_gem_object *obj; @@ -1994,6 +2100,54 @@ int intel_alloc_ringbuffer_obj(struct drm_device *dev, return 0; } +struct intel_ringbuffer * +intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size) +{ + struct intel_ringbuffer *ring; + int ret; + + ring = kzalloc(sizeof(*ring), GFP_KERNEL); + if (ring == NULL) { + DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n", + engine->name); + return ERR_PTR(-ENOMEM); + } + + ring->ring = engine; + list_add(&ring->link, &engine->buffers); + + ring->size = size; + /* Workaround an erratum on the i830 which causes a hang if + * the TAIL pointer points to within the last 2 cachelines + * of the buffer. + */ + ring->effective_size = size; + if (IS_I830(engine->dev) || IS_845G(engine->dev)) + ring->effective_size -= 2 * CACHELINE_BYTES; + + ring->last_retired_head = -1; + intel_ring_update_space(ring); + + ret = intel_alloc_ringbuffer_obj(engine->dev, ring); + if (ret) { + DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n", + engine->name, ret); + list_del(&ring->link); + kfree(ring); + return ERR_PTR(ret); + } + + return ring; +} + +void +intel_ringbuffer_free(struct intel_ringbuffer *ring) +{ + intel_destroy_ringbuffer_obj(ring); + list_del(&ring->link); + kfree(ring); +} + static int intel_init_ring_buffer(struct drm_device *dev, struct intel_engine_cs *ring) { @@ -2002,22 +2156,21 @@ static int intel_init_ring_buffer(struct drm_device *dev, WARN_ON(ring->buffer); - ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); - if (!ringbuf) - return -ENOMEM; - ring->buffer = ringbuf; - ring->dev = dev; INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); INIT_LIST_HEAD(&ring->execlist_queue); + INIT_LIST_HEAD(&ring->buffers); i915_gem_batch_pool_init(dev, &ring->batch_pool); - ringbuf->size = 32 * PAGE_SIZE; - ringbuf->ring = ring; memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno)); init_waitqueue_head(&ring->irq_queue); + ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE); + if (IS_ERR(ringbuf)) + return PTR_ERR(ringbuf); + ring->buffer = ringbuf; + if (I915_NEED_GFX_HWS(dev)) { ret = init_status_page(ring); if (ret) @@ -2029,15 +2182,6 @@ static int intel_init_ring_buffer(struct drm_device *dev, goto error; } - WARN_ON(ringbuf->obj); - - ret = intel_alloc_ringbuffer_obj(dev, ringbuf); - if (ret) { - DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", - ring->name, ret); - goto error; - } - ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf); if (ret) { DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n", @@ -2046,14 +2190,6 @@ static int intel_init_ring_buffer(struct drm_device *dev, goto error; } - /* Workaround an erratum on the i830 which causes a hang if - * the TAIL pointer points to within the last 2 cachelines - * of the buffer. - */ - ringbuf->effective_size = ringbuf->size; - if (IS_I830(dev) || IS_845G(dev)) - ringbuf->effective_size -= 2 * CACHELINE_BYTES; - ret = i915_cmd_parser_init_ring(ring); if (ret) goto error; @@ -2061,7 +2197,7 @@ static int intel_init_ring_buffer(struct drm_device *dev, return 0; error: - kfree(ringbuf); + intel_ringbuffer_free(ringbuf); ring->buffer = NULL; return ret; } @@ -2069,20 +2205,18 @@ error: void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) { struct drm_i915_private *dev_priv; - struct intel_ringbuffer *ringbuf; if (!intel_ring_initialized(ring)) return; dev_priv = to_i915(ring->dev); - ringbuf = ring->buffer; intel_stop_ring_buffer(ring); WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); - intel_unpin_ringbuffer_obj(ringbuf); - intel_destroy_ringbuffer_obj(ringbuf); - i915_gem_request_assign(&ring->outstanding_lazy_request, NULL); + intel_unpin_ringbuffer_obj(ring->buffer); + intel_ringbuffer_free(ring->buffer); + ring->buffer = NULL; if (ring->cleanup) ring->cleanup(ring); @@ -2091,9 +2225,6 @@ void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) i915_cmd_parser_fini_ring(ring); i915_gem_batch_pool_fini(&ring->batch_pool); - - kfree(ringbuf); - ring->buffer = NULL; } static int ring_wait_for_space(struct intel_engine_cs *ring, int n) @@ -2106,6 +2237,9 @@ static int ring_wait_for_space(struct intel_engine_cs *ring, int n) if (intel_ring_space(ringbuf) >= n) return 0; + /* The whole point of reserving space is to not wait! */ + WARN_ON(ringbuf->reserved_in_use); + list_for_each_entry(request, &ring->request_list, list) { space = __intel_ring_space(request->postfix, ringbuf->tail, ringbuf->size); @@ -2124,18 +2258,11 @@ static int ring_wait_for_space(struct intel_engine_cs *ring, int n) return 0; } -static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) +static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf) { uint32_t __iomem *virt; - struct intel_ringbuffer *ringbuf = ring->buffer; int rem = ringbuf->size - ringbuf->tail; - if (ringbuf->space < rem) { - int ret = ring_wait_for_space(ring, rem); - if (ret) - return ret; - } - virt = ringbuf->virtual_start + ringbuf->tail; rem /= 4; while (rem--) @@ -2143,21 +2270,11 @@ static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) ringbuf->tail = 0; intel_ring_update_space(ringbuf); - - return 0; } int intel_ring_idle(struct intel_engine_cs *ring) { struct drm_i915_gem_request *req; - int ret; - - /* We need to add any requests required to flush the objects and ring */ - if (ring->outstanding_lazy_request) { - ret = i915_add_request(ring); - if (ret) - return ret; - } /* Wait upon the last request to be completed */ if (list_empty(&ring->request_list)) @@ -2180,33 +2297,126 @@ int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request) return 0; } -static int __intel_ring_prepare(struct intel_engine_cs *ring, - int bytes) +int intel_ring_reserve_space(struct drm_i915_gem_request *request) +{ + /* + * The first call merely notes the reserve request and is common for + * all back ends. The subsequent localised _begin() call actually + * ensures that the reservation is available. Without the begin, if + * the request creator immediately submitted the request without + * adding any commands to it then there might not actually be + * sufficient room for the submission commands. + */ + intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); + + return intel_ring_begin(request, 0); +} + +void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size) +{ + WARN_ON(ringbuf->reserved_size); + WARN_ON(ringbuf->reserved_in_use); + + ringbuf->reserved_size = size; +} + +void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf) +{ + WARN_ON(ringbuf->reserved_in_use); + + ringbuf->reserved_size = 0; + ringbuf->reserved_in_use = false; +} + +void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf) +{ + WARN_ON(ringbuf->reserved_in_use); + + ringbuf->reserved_in_use = true; + ringbuf->reserved_tail = ringbuf->tail; +} + +void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf) +{ + WARN_ON(!ringbuf->reserved_in_use); + if (ringbuf->tail > ringbuf->reserved_tail) { + WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size, + "request reserved size too small: %d vs %d!\n", + ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size); + } else { + /* + * The ring was wrapped while the reserved space was in use. + * That means that some unknown amount of the ring tail was + * no-op filled and skipped. Thus simply adding the ring size + * to the tail and doing the above space check will not work. + * Rather than attempt to track how much tail was skipped, + * it is much simpler to say that also skipping the sanity + * check every once in a while is not a big issue. + */ + } + + ringbuf->reserved_size = 0; + ringbuf->reserved_in_use = false; +} + +static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes) { struct intel_ringbuffer *ringbuf = ring->buffer; - int ret; + int remain_usable = ringbuf->effective_size - ringbuf->tail; + int remain_actual = ringbuf->size - ringbuf->tail; + int ret, total_bytes, wait_bytes = 0; + bool need_wrap = false; - if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) { - ret = intel_wrap_ring_buffer(ring); - if (unlikely(ret)) - return ret; + if (ringbuf->reserved_in_use) + total_bytes = bytes; + else + total_bytes = bytes + ringbuf->reserved_size; + + if (unlikely(bytes > remain_usable)) { + /* + * Not enough space for the basic request. So need to flush + * out the remainder and then wait for base + reserved. + */ + wait_bytes = remain_actual + total_bytes; + need_wrap = true; + } else { + if (unlikely(total_bytes > remain_usable)) { + /* + * The base request will fit but the reserved space + * falls off the end. So only need to to wait for the + * reserved size after flushing out the remainder. + */ + wait_bytes = remain_actual + ringbuf->reserved_size; + need_wrap = true; + } else if (total_bytes > ringbuf->space) { + /* No wrapping required, just waiting. */ + wait_bytes = total_bytes; + } } - if (unlikely(ringbuf->space < bytes)) { - ret = ring_wait_for_space(ring, bytes); + if (wait_bytes) { + ret = ring_wait_for_space(ring, wait_bytes); if (unlikely(ret)) return ret; + + if (need_wrap) + __wrap_ring_buffer(ringbuf); } return 0; } -int intel_ring_begin(struct intel_engine_cs *ring, +int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct intel_engine_cs *ring; + struct drm_i915_private *dev_priv; int ret; + WARN_ON(req == NULL); + ring = req->ring; + dev_priv = ring->dev->dev_private; + ret = i915_gem_check_wedge(&dev_priv->gpu_error, dev_priv->mm.interruptible); if (ret) @@ -2216,18 +2426,14 @@ int intel_ring_begin(struct intel_engine_cs *ring, if (ret) return ret; - /* Preallocate the olr before touching the ring */ - ret = i915_gem_request_alloc(ring, ring->default_context); - if (ret) - return ret; - ring->buffer->space -= num_dwords * sizeof(uint32_t); return 0; } /* Align the ring tail to a cacheline boundary */ -int intel_ring_cacheline_align(struct intel_engine_cs *ring) +int intel_ring_cacheline_align(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); int ret; @@ -2235,7 +2441,7 @@ int intel_ring_cacheline_align(struct intel_engine_cs *ring) return 0; num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords; - ret = intel_ring_begin(ring, num_dwords); + ret = intel_ring_begin(req, num_dwords); if (ret) return ret; @@ -2252,8 +2458,6 @@ void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno) struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; - BUG_ON(ring->outstanding_lazy_request); - if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) { I915_WRITE(RING_SYNC_0(ring->mmio_base), 0); I915_WRITE(RING_SYNC_1(ring->mmio_base), 0); @@ -2298,13 +2502,14 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); } -static int gen6_bsd_ring_flush(struct intel_engine_cs *ring, +static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 invalidate, u32 flush) { + struct intel_engine_cs *ring = req->ring; uint32_t cmd; int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -2342,20 +2547,23 @@ static int gen6_bsd_ring_flush(struct intel_engine_cs *ring, } static int -gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring, +gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req, u64 offset, u32 len, unsigned dispatch_flags) { + struct intel_engine_cs *ring = req->ring; bool ppgtt = USES_PPGTT(ring->dev) && !(dispatch_flags & I915_DISPATCH_SECURE); int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; /* FIXME(BDW): Address space and security selectors. */ - intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); + intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) | + (dispatch_flags & I915_DISPATCH_RS ? + MI_BATCH_RESOURCE_STREAMER : 0)); intel_ring_emit(ring, lower_32_bits(offset)); intel_ring_emit(ring, upper_32_bits(offset)); intel_ring_emit(ring, MI_NOOP); @@ -2365,20 +2573,23 @@ gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring, } static int -hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, +hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req, u64 offset, u32 len, unsigned dispatch_flags) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; intel_ring_emit(ring, MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? - 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW)); + 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) | + (dispatch_flags & I915_DISPATCH_RS ? + MI_BATCH_RESOURCE_STREAMER : 0)); /* bit0-7 is the length on GEN6+ */ intel_ring_emit(ring, offset); intel_ring_advance(ring); @@ -2387,13 +2598,14 @@ hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, } static int -gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, +gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req, u64 offset, u32 len, unsigned dispatch_flags) { + struct intel_engine_cs *ring = req->ring; int ret; - ret = intel_ring_begin(ring, 2); + ret = intel_ring_begin(req, 2); if (ret) return ret; @@ -2410,14 +2622,15 @@ gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, /* Blitter support (SandyBridge+) */ -static int gen6_ring_flush(struct intel_engine_cs *ring, +static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 invalidate, u32 flush) { + struct intel_engine_cs *ring = req->ring; struct drm_device *dev = ring->dev; uint32_t cmd; int ret; - ret = intel_ring_begin(ring, 4); + ret = intel_ring_begin(req, 4); if (ret) return ret; @@ -2498,6 +2711,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev) GEN8_RING_SEMAPHORE_INIT; } } else if (INTEL_INFO(dev)->gen >= 6) { + ring->init_context = intel_rcs_ctx_init; ring->add_request = gen6_add_request; ring->flush = gen7_render_ring_flush; if (INTEL_INFO(dev)->gen == 6) @@ -2818,26 +3032,28 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev) } int -intel_ring_flush_all_caches(struct intel_engine_cs *ring) +intel_ring_flush_all_caches(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; int ret; if (!ring->gpu_caches_dirty) return 0; - ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS); + ret = ring->flush(req, 0, I915_GEM_GPU_DOMAINS); if (ret) return ret; - trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS); + trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS); ring->gpu_caches_dirty = false; return 0; } int -intel_ring_invalidate_all_caches(struct intel_engine_cs *ring) +intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req) { + struct intel_engine_cs *ring = req->ring; uint32_t flush_domains; int ret; @@ -2845,11 +3061,11 @@ intel_ring_invalidate_all_caches(struct intel_engine_cs *ring) if (ring->gpu_caches_dirty) flush_domains = I915_GEM_GPU_DOMAINS; - ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); + ret = ring->flush(req, I915_GEM_GPU_DOMAINS, flush_domains); if (ret) return ret; - trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains); + trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains); ring->gpu_caches_dirty = false; return 0; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 4be66f60504d..5d1eb206151d 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -12,6 +12,7 @@ * workarounds! */ #define CACHELINE_BYTES 64 +#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t)) /* * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use" @@ -99,12 +100,16 @@ struct intel_ringbuffer { void __iomem *virtual_start; struct intel_engine_cs *ring; + struct list_head link; u32 head; u32 tail; int space; int size; int effective_size; + int reserved_size; + int reserved_tail; + bool reserved_in_use; /** We track the position of the requests in the ring buffer, and * when each is retired we increment last_retired_head as the GPU @@ -120,6 +125,25 @@ struct intel_ringbuffer { struct intel_context; struct drm_i915_reg_descriptor; +/* + * we use a single page to load ctx workarounds so all of these + * values are referred in terms of dwords + * + * struct i915_wa_ctx_bb: + * offset: specifies batch starting position, also helpful in case + * if we want to have multiple batches at different offsets based on + * some criteria. It is not a requirement at the moment but provides + * an option for future use. + * size: size of the batch in DWORDS + */ +struct i915_ctx_workarounds { + struct i915_wa_ctx_bb { + u32 offset; + u32 size; + } indirect_ctx, per_ctx; + struct drm_i915_gem_object *obj; +}; + struct intel_engine_cs { const char *name; enum intel_ring_id { @@ -134,6 +158,7 @@ struct intel_engine_cs { u32 mmio_base; struct drm_device *dev; struct intel_ringbuffer *buffer; + struct list_head buffers; /* * A pool of objects to use as shadow copies of client batch buffers @@ -143,6 +168,7 @@ struct intel_engine_cs { struct i915_gem_batch_pool batch_pool; struct intel_hw_status_page status_page; + struct i915_ctx_workarounds wa_ctx; unsigned irq_refcount; /* protected by dev_priv->irq_lock */ u32 irq_enable_mask; /* bitmask to enable ring interrupt */ @@ -152,15 +178,14 @@ struct intel_engine_cs { int (*init_hw)(struct intel_engine_cs *ring); - int (*init_context)(struct intel_engine_cs *ring, - struct intel_context *ctx); + int (*init_context)(struct drm_i915_gem_request *req); void (*write_tail)(struct intel_engine_cs *ring, u32 value); - int __must_check (*flush)(struct intel_engine_cs *ring, + int __must_check (*flush)(struct drm_i915_gem_request *req, u32 invalidate_domains, u32 flush_domains); - int (*add_request)(struct intel_engine_cs *ring); + int (*add_request)(struct drm_i915_gem_request *req); /* Some chipsets are not quite as coherent as advertised and need * an expensive kick to force a true read of the up-to-date seqno. * However, the up-to-date seqno is not always required and the last @@ -171,11 +196,12 @@ struct intel_engine_cs { bool lazy_coherency); void (*set_seqno)(struct intel_engine_cs *ring, u32 seqno); - int (*dispatch_execbuffer)(struct intel_engine_cs *ring, + int (*dispatch_execbuffer)(struct drm_i915_gem_request *req, u64 offset, u32 length, unsigned dispatch_flags); #define I915_DISPATCH_SECURE 0x1 #define I915_DISPATCH_PINNED 0x2 +#define I915_DISPATCH_RS 0x4 void (*cleanup)(struct intel_engine_cs *ring); /* GEN8 signal/wait table - never trust comments! @@ -223,16 +249,16 @@ struct intel_engine_cs { /* our mbox written by others */ u32 wait[I915_NUM_RINGS]; /* mboxes this ring signals to */ - u32 signal[I915_NUM_RINGS]; + i915_reg_t signal[I915_NUM_RINGS]; } mbox; u64 signal_ggtt[I915_NUM_RINGS]; }; /* AKA wait() */ - int (*sync_to)(struct intel_engine_cs *ring, - struct intel_engine_cs *to, + int (*sync_to)(struct drm_i915_gem_request *to_req, + struct intel_engine_cs *from, u32 seqno); - int (*signal)(struct intel_engine_cs *signaller, + int (*signal)(struct drm_i915_gem_request *signaller_req, /* num_dwords needed by caller */ unsigned int num_dwords); } semaphore; @@ -243,14 +269,11 @@ struct intel_engine_cs { struct list_head execlist_retired_req_list; u8 next_context_status_buffer; u32 irq_keep_mask; /* bitmask for interrupts that should not be masked */ - int (*emit_request)(struct intel_ringbuffer *ringbuf, - struct drm_i915_gem_request *request); - int (*emit_flush)(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, + int (*emit_request)(struct drm_i915_gem_request *request); + int (*emit_flush)(struct drm_i915_gem_request *request, u32 invalidate_domains, u32 flush_domains); - int (*emit_bb_start)(struct intel_ringbuffer *ringbuf, - struct intel_context *ctx, + int (*emit_bb_start)(struct drm_i915_gem_request *req, u64 offset, unsigned dispatch_flags); /** @@ -272,10 +295,6 @@ struct intel_engine_cs { struct list_head request_list; /** - * Do we have some not yet emitted requests outstanding? - */ - struct drm_i915_gem_request *outstanding_lazy_request; - /** * Seqno of request most recently submitted to request_list. * Used exclusively by hang checker to avoid grabbing lock while * inspecting request list. @@ -360,6 +379,13 @@ intel_ring_sync_index(struct intel_engine_cs *ring, return idx; } +static inline void +intel_flush_status_page(struct intel_engine_cs *ring, int reg) +{ + drm_clflush_virt_range(&ring->status_page.page_addr[reg], + sizeof(uint32_t)); +} + static inline u32 intel_read_status_page(struct intel_engine_cs *ring, int reg) @@ -396,20 +422,20 @@ intel_write_status_page(struct intel_engine_cs *ring, #define I915_GEM_HWS_SCRATCH_INDEX 0x40 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT) -void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf); +struct intel_ringbuffer * +intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size); int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, struct intel_ringbuffer *ringbuf); -void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf); -int intel_alloc_ringbuffer_obj(struct drm_device *dev, - struct intel_ringbuffer *ringbuf); +void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf); +void intel_ringbuffer_free(struct intel_ringbuffer *ring); void intel_stop_ring_buffer(struct intel_engine_cs *ring); void intel_cleanup_ring_buffer(struct intel_engine_cs *ring); int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request); -int __must_check intel_ring_begin(struct intel_engine_cs *ring, int n); -int __must_check intel_ring_cacheline_align(struct intel_engine_cs *ring); +int __must_check intel_ring_begin(struct drm_i915_gem_request *req, int n); +int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req); static inline void intel_ring_emit(struct intel_engine_cs *ring, u32 data) { @@ -417,6 +443,11 @@ static inline void intel_ring_emit(struct intel_engine_cs *ring, iowrite32(data, ringbuf->virtual_start + ringbuf->tail); ringbuf->tail += 4; } +static inline void intel_ring_emit_reg(struct intel_engine_cs *ring, + i915_reg_t reg) +{ + intel_ring_emit(ring, i915_mmio_reg_offset(reg)); +} static inline void intel_ring_advance(struct intel_engine_cs *ring) { struct intel_ringbuffer *ringbuf = ring->buffer; @@ -426,12 +457,11 @@ int __intel_ring_space(int head, int tail, int size); void intel_ring_update_space(struct intel_ringbuffer *ringbuf); int intel_ring_space(struct intel_ringbuffer *ringbuf); bool intel_ring_stopped(struct intel_engine_cs *ring); -void __intel_ring_advance(struct intel_engine_cs *ring); int __must_check intel_ring_idle(struct intel_engine_cs *ring); void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno); -int intel_ring_flush_all_caches(struct intel_engine_cs *ring); -int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring); +int intel_ring_flush_all_caches(struct drm_i915_gem_request *req); +int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req); void intel_fini_pipe_control(struct intel_engine_cs *ring); int intel_init_pipe_control(struct intel_engine_cs *ring); @@ -451,11 +481,29 @@ static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf) return ringbuf->tail; } -static inline struct drm_i915_gem_request * -intel_ring_get_request(struct intel_engine_cs *ring) -{ - BUG_ON(ring->outstanding_lazy_request == NULL); - return ring->outstanding_lazy_request; -} +/* + * Arbitrary size for largest possible 'add request' sequence. The code paths + * are complex and variable. Empirical measurement shows that the worst case + * is ILK at 136 words. Reserving too much is better than reserving too little + * as that allows for corner cases that might have been missed. So the figure + * has been rounded up to 160 words. + */ +#define MIN_SPACE_FOR_ADD_REQUEST 160 + +/* + * Reserve space in the ring to guarantee that the i915_add_request() call + * will always have sufficient room to do its stuff. The request creation + * code calls this automatically. + */ +void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size); +/* Cancel the reservation, e.g. because the request is being discarded. */ +void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf); +/* Use the reserved space - for use by i915_add_request() only. */ +void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf); +/* Finish with the reserved space - for use by i915_add_request() only. */ +void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf); + +/* Legacy ringbuffer specific portion of reservation code: */ +int intel_ring_reserve_space(struct drm_i915_gem_request *request); #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 1a45385f4d66..2c2151f1c47e 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -49,25 +49,104 @@ * present for a given platform. */ -#define GEN9_ENABLE_DC5(dev) 0 -#define SKL_ENABLE_DC6(dev) IS_SKYLAKE(dev) - #define for_each_power_well(i, power_well, domain_mask, power_domains) \ for (i = 0; \ i < (power_domains)->power_well_count && \ ((power_well) = &(power_domains)->power_wells[i]); \ i++) \ - if ((power_well)->domains & (domain_mask)) + for_each_if ((power_well)->domains & (domain_mask)) #define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \ for (i = (power_domains)->power_well_count - 1; \ i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\ i--) \ - if ((power_well)->domains & (domain_mask)) + for_each_if ((power_well)->domains & (domain_mask)) bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, int power_well_id); +const char * +intel_display_power_domain_str(enum intel_display_power_domain domain) +{ + switch (domain) { + case POWER_DOMAIN_PIPE_A: + return "PIPE_A"; + case POWER_DOMAIN_PIPE_B: + return "PIPE_B"; + case POWER_DOMAIN_PIPE_C: + return "PIPE_C"; + case POWER_DOMAIN_PIPE_A_PANEL_FITTER: + return "PIPE_A_PANEL_FITTER"; + case POWER_DOMAIN_PIPE_B_PANEL_FITTER: + return "PIPE_B_PANEL_FITTER"; + case POWER_DOMAIN_PIPE_C_PANEL_FITTER: + return "PIPE_C_PANEL_FITTER"; + case POWER_DOMAIN_TRANSCODER_A: + return "TRANSCODER_A"; + case POWER_DOMAIN_TRANSCODER_B: + return "TRANSCODER_B"; + case POWER_DOMAIN_TRANSCODER_C: + return "TRANSCODER_C"; + case POWER_DOMAIN_TRANSCODER_EDP: + return "TRANSCODER_EDP"; + case POWER_DOMAIN_PORT_DDI_A_LANES: + return "PORT_DDI_A_LANES"; + case POWER_DOMAIN_PORT_DDI_B_LANES: + return "PORT_DDI_B_LANES"; + case POWER_DOMAIN_PORT_DDI_C_LANES: + return "PORT_DDI_C_LANES"; + case POWER_DOMAIN_PORT_DDI_D_LANES: + return "PORT_DDI_D_LANES"; + case POWER_DOMAIN_PORT_DDI_E_LANES: + return "PORT_DDI_E_LANES"; + case POWER_DOMAIN_PORT_DSI: + return "PORT_DSI"; + case POWER_DOMAIN_PORT_CRT: + return "PORT_CRT"; + case POWER_DOMAIN_PORT_OTHER: + return "PORT_OTHER"; + case POWER_DOMAIN_VGA: + return "VGA"; + case POWER_DOMAIN_AUDIO: + return "AUDIO"; + case POWER_DOMAIN_PLLS: + return "PLLS"; + case POWER_DOMAIN_AUX_A: + return "AUX_A"; + case POWER_DOMAIN_AUX_B: + return "AUX_B"; + case POWER_DOMAIN_AUX_C: + return "AUX_C"; + case POWER_DOMAIN_AUX_D: + return "AUX_D"; + case POWER_DOMAIN_GMBUS: + return "GMBUS"; + case POWER_DOMAIN_INIT: + return "INIT"; + case POWER_DOMAIN_MODESET: + return "MODESET"; + default: + MISSING_CASE(domain); + return "?"; + } +} + +static void intel_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + DRM_DEBUG_KMS("enabling %s\n", power_well->name); + power_well->ops->enable(dev_priv, power_well); + power_well->hw_enabled = true; +} + +static void intel_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + DRM_DEBUG_KMS("disabling %s\n", power_well->name); + power_well->hw_enabled = false; + power_well->ops->disable(dev_priv, power_well); +} + /* * We should only use the power well if we explicitly asked the hardware to * enable it, so check if it's enabled and also check if we've requested it to @@ -228,11 +307,6 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv, gen8_irq_power_well_post_enable(dev_priv, 1 << PIPE_C | 1 << PIPE_B); } - - if (power_well->data == SKL_DISP_PW_1) { - intel_prepare_ddi(dev); - gen8_irq_power_well_post_enable(dev_priv, 1 << PIPE_A); - } } static void hsw_set_power_well(struct drm_i915_private *dev_priv, @@ -275,56 +349,38 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv, BIT(POWER_DOMAIN_TRANSCODER_C) | \ BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \ BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_E_LANES) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_AUX_D) | \ BIT(POWER_DOMAIN_AUDIO) | \ BIT(POWER_DOMAIN_VGA) | \ BIT(POWER_DOMAIN_INIT)) -#define SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS ( \ - SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS | \ - BIT(POWER_DOMAIN_PLLS) | \ - BIT(POWER_DOMAIN_PIPE_A) | \ - BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ - BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ - BIT(POWER_DOMAIN_AUX_A) | \ - BIT(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_DDI_A_E_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_E_LANES) | \ BIT(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_DDI_B_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ BIT(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_DDI_C_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_DDI_D_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_LANES) | \ BIT(POWER_DOMAIN_INIT)) -#define SKL_DISPLAY_MISC_IO_POWER_DOMAINS ( \ - SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS | \ - BIT(POWER_DOMAIN_PLLS) | \ +#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS ( \ + SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS | \ + BIT(POWER_DOMAIN_MODESET) | \ + BIT(POWER_DOMAIN_AUX_A) | \ BIT(POWER_DOMAIN_INIT)) #define SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS ( \ - (POWER_DOMAIN_MASK & ~(SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS | \ + (POWER_DOMAIN_MASK & ~( \ SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS | \ - SKL_DISPLAY_DDI_A_E_POWER_DOMAINS | \ - SKL_DISPLAY_DDI_B_POWER_DOMAINS | \ - SKL_DISPLAY_DDI_C_POWER_DOMAINS | \ - SKL_DISPLAY_DDI_D_POWER_DOMAINS | \ - SKL_DISPLAY_MISC_IO_POWER_DOMAINS)) | \ + SKL_DISPLAY_DC_OFF_POWER_DOMAINS)) | \ BIT(POWER_DOMAIN_INIT)) #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS ( \ @@ -335,25 +391,28 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv, BIT(POWER_DOMAIN_TRANSCODER_C) | \ BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) | \ BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_AUDIO) | \ BIT(POWER_DOMAIN_VGA) | \ + BIT(POWER_DOMAIN_GMBUS) | \ BIT(POWER_DOMAIN_INIT)) #define BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS ( \ BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS | \ BIT(POWER_DOMAIN_PIPE_A) | \ BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_LANES) | \ BIT(POWER_DOMAIN_AUX_A) | \ BIT(POWER_DOMAIN_PLLS) | \ BIT(POWER_DOMAIN_INIT)) +#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS ( \ + BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS | \ + BIT(POWER_DOMAIN_MODESET) | \ + BIT(POWER_DOMAIN_AUX_A) | \ + BIT(POWER_DOMAIN_INIT)) #define BXT_DISPLAY_ALWAYS_ON_POWER_DOMAINS ( \ (POWER_DOMAIN_MASK & ~(BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS | \ BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS)) | \ @@ -397,46 +456,74 @@ static void assert_can_disable_dc9(struct drm_i915_private *dev_priv) */ } -void bxt_enable_dc9(struct drm_i915_private *dev_priv) +static void gen9_set_dc_state_debugmask_memory_up( + struct drm_i915_private *dev_priv) { uint32_t val; - assert_can_enable_dc9(dev_priv); + /* The below bit doesn't need to be cleared ever afterwards */ + val = I915_READ(DC_STATE_DEBUG); + if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) { + val |= DC_STATE_DEBUG_MASK_MEMORY_UP; + I915_WRITE(DC_STATE_DEBUG, val); + POSTING_READ(DC_STATE_DEBUG); + } +} - DRM_DEBUG_KMS("Enabling DC9\n"); +static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state) +{ + uint32_t val; + uint32_t mask; + + mask = DC_STATE_EN_UPTO_DC5; + if (IS_BROXTON(dev_priv)) + mask |= DC_STATE_EN_DC9; + else + mask |= DC_STATE_EN_UPTO_DC6; + + WARN_ON_ONCE(state & ~mask); + + if (i915.enable_dc == 0) + state = DC_STATE_DISABLE; + else if (i915.enable_dc == 1 && state > DC_STATE_EN_UPTO_DC5) + state = DC_STATE_EN_UPTO_DC5; + + if (state & DC_STATE_EN_UPTO_DC5_DC6_MASK) + gen9_set_dc_state_debugmask_memory_up(dev_priv); val = I915_READ(DC_STATE_EN); - val |= DC_STATE_EN_DC9; + DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n", + val & mask, state); + val &= ~mask; + val |= state; I915_WRITE(DC_STATE_EN, val); POSTING_READ(DC_STATE_EN); } -void bxt_disable_dc9(struct drm_i915_private *dev_priv) +void bxt_enable_dc9(struct drm_i915_private *dev_priv) { - uint32_t val; + assert_can_enable_dc9(dev_priv); + + DRM_DEBUG_KMS("Enabling DC9\n"); + + gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9); +} +void bxt_disable_dc9(struct drm_i915_private *dev_priv) +{ assert_can_disable_dc9(dev_priv); DRM_DEBUG_KMS("Disabling DC9\n"); - val = I915_READ(DC_STATE_EN); - val &= ~DC_STATE_EN_DC9; - I915_WRITE(DC_STATE_EN, val); - POSTING_READ(DC_STATE_EN); + gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); } -static void gen9_set_dc_state_debugmask_memory_up( - struct drm_i915_private *dev_priv) +static void assert_csr_loaded(struct drm_i915_private *dev_priv) { - uint32_t val; - - /* The below bit doesn't need to be cleared ever afterwards */ - val = I915_READ(DC_STATE_DEBUG); - if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) { - val |= DC_STATE_DEBUG_MASK_MEMORY_UP; - I915_WRITE(DC_STATE_DEBUG, val); - POSTING_READ(DC_STATE_DEBUG); - } + WARN_ONCE(!I915_READ(CSR_PROGRAM(0)), + "CSR program storage start is NULL\n"); + WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n"); + WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n"); } static void assert_can_enable_dc5(struct drm_i915_private *dev_priv) @@ -445,22 +532,20 @@ static void assert_can_enable_dc5(struct drm_i915_private *dev_priv) bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv, SKL_DISP_PW_2); - WARN(!IS_SKYLAKE(dev), "Platform doesn't support DC5.\n"); - WARN(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n"); - WARN(pg2_enabled, "PG2 not disabled to enable DC5.\n"); + WARN_ONCE(!IS_SKYLAKE(dev), "Platform doesn't support DC5.\n"); + WARN_ONCE(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n"); + WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n"); - WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5), - "DC5 already programmed to be enabled.\n"); - WARN(dev_priv->pm.suspended, - "DC5 cannot be enabled, if platform is runtime-suspended.\n"); + WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5), + "DC5 already programmed to be enabled.\n"); + WARN_ONCE(dev_priv->pm.suspended, + "DC5 cannot be enabled, if platform is runtime-suspended.\n"); assert_csr_loaded(dev_priv); } static void assert_can_disable_dc5(struct drm_i915_private *dev_priv) { - bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv, - SKL_DISP_PW_2); /* * During initialization, the firmware may not be loaded yet. * We still want to make sure that the DC enabling flag is cleared. @@ -468,52 +553,29 @@ static void assert_can_disable_dc5(struct drm_i915_private *dev_priv) if (dev_priv->power_domains.initializing) return; - WARN(!pg2_enabled, "PG2 not enabled to disable DC5.\n"); - WARN(dev_priv->pm.suspended, + WARN_ONCE(dev_priv->pm.suspended, "Disabling of DC5 while platform is runtime-suspended should never happen.\n"); } static void gen9_enable_dc5(struct drm_i915_private *dev_priv) { - uint32_t val; - assert_can_enable_dc5(dev_priv); DRM_DEBUG_KMS("Enabling DC5\n"); - gen9_set_dc_state_debugmask_memory_up(dev_priv); - - val = I915_READ(DC_STATE_EN); - val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK; - val |= DC_STATE_EN_UPTO_DC5; - I915_WRITE(DC_STATE_EN, val); - POSTING_READ(DC_STATE_EN); -} - -static void gen9_disable_dc5(struct drm_i915_private *dev_priv) -{ - uint32_t val; - - assert_can_disable_dc5(dev_priv); - - DRM_DEBUG_KMS("Disabling DC5\n"); - - val = I915_READ(DC_STATE_EN); - val &= ~DC_STATE_EN_UPTO_DC5; - I915_WRITE(DC_STATE_EN, val); - POSTING_READ(DC_STATE_EN); + gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5); } static void assert_can_enable_dc6(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; - WARN(!IS_SKYLAKE(dev), "Platform doesn't support DC6.\n"); - WARN(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n"); - WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE, - "Backlight is not disabled.\n"); - WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6), - "DC6 already programmed to be enabled.\n"); + WARN_ONCE(!IS_SKYLAKE(dev), "Platform doesn't support DC6.\n"); + WARN_ONCE(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n"); + WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE, + "Backlight is not disabled.\n"); + WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6), + "DC6 already programmed to be enabled.\n"); assert_csr_loaded(dev_priv); } @@ -527,40 +589,37 @@ static void assert_can_disable_dc6(struct drm_i915_private *dev_priv) if (dev_priv->power_domains.initializing) return; - assert_csr_loaded(dev_priv); - WARN(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6), - "DC6 already programmed to be disabled.\n"); + WARN_ONCE(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6), + "DC6 already programmed to be disabled.\n"); } -static void skl_enable_dc6(struct drm_i915_private *dev_priv) +static void gen9_disable_dc5_dc6(struct drm_i915_private *dev_priv) { - uint32_t val; + assert_can_disable_dc5(dev_priv); + if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && i915.enable_dc != 1) + assert_can_disable_dc6(dev_priv); + + gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); +} + +void skl_enable_dc6(struct drm_i915_private *dev_priv) +{ assert_can_enable_dc6(dev_priv); DRM_DEBUG_KMS("Enabling DC6\n"); - gen9_set_dc_state_debugmask_memory_up(dev_priv); + gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6); - val = I915_READ(DC_STATE_EN); - val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK; - val |= DC_STATE_EN_UPTO_DC6; - I915_WRITE(DC_STATE_EN, val); - POSTING_READ(DC_STATE_EN); } -static void skl_disable_dc6(struct drm_i915_private *dev_priv) +void skl_disable_dc6(struct drm_i915_private *dev_priv) { - uint32_t val; - assert_can_disable_dc6(dev_priv); DRM_DEBUG_KMS("Disabling DC6\n"); - val = I915_READ(DC_STATE_EN); - val &= ~DC_STATE_EN_UPTO_DC6; - I915_WRITE(DC_STATE_EN, val); - POSTING_READ(DC_STATE_EN); + gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); } static void skl_set_power_well(struct drm_i915_private *dev_priv, @@ -610,20 +669,16 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, !I915_READ(HSW_PWR_WELL_BIOS), "Invalid for power well status to be enabled, unless done by the BIOS, \ when request is to disable!\n"); - if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) && - power_well->data == SKL_DISP_PW_2) { - if (SKL_ENABLE_DC6(dev)) { - skl_disable_dc6(dev_priv); - /* - * DDI buffer programming unnecessary during driver-load/resume - * as it's already done during modeset initialization then. - * It's also invalid here as encoder list is still uninitialized. - */ - if (!dev_priv->power_domains.initializing) - intel_prepare_ddi(dev); - } else { - gen9_disable_dc5(dev_priv); - } + if (power_well->data == SKL_DISP_PW_2) { + /* + * DDI buffer programming unnecessary during + * driver-load/resume as it's already done + * during modeset initialization then. It's + * also invalid here as encoder list is still + * uninitialized. + */ + if (!dev_priv->power_domains.initializing) + intel_prepare_ddi(dev); } I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask); } @@ -641,25 +696,6 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask); POSTING_READ(HSW_PWR_WELL_DRIVER); DRM_DEBUG_KMS("Disabling %s\n", power_well->name); - - if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) && - power_well->data == SKL_DISP_PW_2) { - enum csr_state state; - /* TODO: wait for a completion event or - * similar here instead of busy - * waiting using wait_for function. - */ - wait_for((state = intel_csr_load_status_get(dev_priv)) != - FW_UNINITIALIZED, 1000); - if (state != FW_LOADED) - DRM_ERROR("CSR firmware not ready (%d)\n", - state); - else - if (SKL_ENABLE_DC6(dev)) - skl_enable_dc6(dev_priv); - else - gen9_enable_dc5(dev_priv); - } } } @@ -734,6 +770,41 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv, skl_set_power_well(dev_priv, power_well, false); } +static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0; +} + +static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + gen9_disable_dc5_dc6(dev_priv); +} + +static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && i915.enable_dc != 1) + skl_enable_dc6(dev_priv); + else + gen9_enable_dc5(dev_priv); +} + +static void gen9_dc_off_power_well_sync_hw(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + if (power_well->count > 0) { + gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); + } else { + if (IS_SKYLAKE(dev_priv) && i915.enable_dc != 0 && + i915.enable_dc != 1) + gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6); + else + gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5); + } +} + static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { @@ -835,12 +906,27 @@ static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, return enabled; } -static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) +static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + enum pipe pipe; - vlv_set_power_well(dev_priv, power_well, true); + /* + * Enable the CRI clock source so we can get at the + * display and the reference clock for VGA + * hotplug / manual detection. Supposedly DSI also + * needs the ref clock up and running. + * + * CHV DPLL B/C have some issues if VGA mode is enabled. + */ + for_each_pipe(dev_priv->dev, pipe) { + u32 val = I915_READ(DPLL(pipe)); + + val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; + if (pipe != PIPE_A) + val |= DPLL_INTEGRATED_CRI_CLK_VLV; + + I915_WRITE(DPLL(pipe), val); + } spin_lock_irq(&dev_priv->irq_lock); valleyview_enable_display_irqs(dev_priv); @@ -858,18 +944,33 @@ static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, i915_redisable_vga_power_on(dev_priv->dev); } +static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + valleyview_disable_display_irqs(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); + + vlv_power_sequencer_reset(dev_priv); +} + +static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + + vlv_set_power_well(dev_priv, power_well, true); + + vlv_display_power_well_init(dev_priv); +} + static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); - spin_lock_irq(&dev_priv->irq_lock); - valleyview_disable_display_irqs(dev_priv); - spin_unlock_irq(&dev_priv->irq_lock); + vlv_display_power_well_deinit(dev_priv); vlv_set_power_well(dev_priv, power_well, false); - - vlv_power_sequencer_reset(dev_priv); } static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, @@ -877,13 +978,7 @@ static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, { WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); - /* - * Enable the CRI clock source so we can get at the - * display and the reference clock for VGA - * hotplug / manual detection. - */ - I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | - DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); + /* since ref/cri clock was enabled */ udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ vlv_set_power_well(dev_priv, power_well, true); @@ -918,30 +1013,151 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, vlv_set_power_well(dev_priv, power_well, false); } +#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) + +static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv, + int power_well_id) +{ + struct i915_power_domains *power_domains = &dev_priv->power_domains; + int i; + + for (i = 0; i < power_domains->power_well_count; i++) { + struct i915_power_well *power_well; + + power_well = &power_domains->power_wells[i]; + if (power_well->data == power_well_id) + return power_well; + } + + return NULL; +} + +#define BITS_SET(val, bits) (((val) & (bits)) == (bits)) + +static void assert_chv_phy_status(struct drm_i915_private *dev_priv) +{ + struct i915_power_well *cmn_bc = + lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC); + struct i915_power_well *cmn_d = + lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D); + u32 phy_control = dev_priv->chv_phy_control; + u32 phy_status = 0; + u32 phy_status_mask = 0xffffffff; + u32 tmp; + + /* + * The BIOS can leave the PHY is some weird state + * where it doesn't fully power down some parts. + * Disable the asserts until the PHY has been fully + * reset (ie. the power well has been disabled at + * least once). + */ + if (!dev_priv->chv_phy_assert[DPIO_PHY0]) + phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) | + PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) | + PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) | + PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) | + PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) | + PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1)); + + if (!dev_priv->chv_phy_assert[DPIO_PHY1]) + phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) | + PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) | + PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1)); + + if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) { + phy_status |= PHY_POWERGOOD(DPIO_PHY0); + + /* this assumes override is only used to enable lanes */ + if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0) + phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0); + + if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0) + phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1); + + /* CL1 is on whenever anything is on in either channel */ + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) | + PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1))) + phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0); + + /* + * The DPLLB check accounts for the pipe B + port A usage + * with CL2 powered up but all the lanes in the second channel + * powered down. + */ + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) && + (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0) + phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1); + + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0))) + phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0); + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0))) + phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1); + + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1))) + phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0); + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1))) + phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1); + } + + if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) { + phy_status |= PHY_POWERGOOD(DPIO_PHY1); + + /* this assumes override is only used to enable lanes */ + if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0) + phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0); + + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0))) + phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0); + + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0))) + phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0); + if (BITS_SET(phy_control, + PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0))) + phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1); + } + + phy_status &= phy_status_mask; + + /* + * The PHY may be busy with some initial calibration and whatnot, + * so the power state can take a while to actually change. + */ + if (wait_for((tmp = I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask) == phy_status, 10)) + WARN(phy_status != tmp, + "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n", + tmp, phy_status, dev_priv->chv_phy_control); +} + +#undef BITS_SET + static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { enum dpio_phy phy; + enum pipe pipe; + uint32_t tmp; WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); - /* - * Enable the CRI clock source so we can get at the - * display and the reference clock for VGA - * hotplug / manual detection. - */ if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + pipe = PIPE_A; phy = DPIO_PHY0; - I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | - DPLL_REFA_CLK_ENABLE_VLV); - I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | - DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); } else { + pipe = PIPE_C; phy = DPIO_PHY1; - I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) | - DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV); } + + /* since ref/cri clock was enabled */ udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ vlv_set_power_well(dev_priv, power_well, true); @@ -949,8 +1165,38 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1)) DRM_ERROR("Display PHY %d is not power up\n", phy); + mutex_lock(&dev_priv->sb_lock); + + /* Enable dynamic power down */ + tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28); + tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN | + DPIO_SUS_CLK_CONFIG_GATE_CLKREQ; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp); + + if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1); + tmp |= DPIO_DYNPWRDOWNEN_CH1; + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp); + } else { + /* + * Force the non-existing CL2 off. BXT does this + * too, so maybe it saves some power even though + * CL2 doesn't exist? + */ + tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30); + tmp |= DPIO_CL2_LDOFUSE_PWRENB; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp); + } + + mutex_unlock(&dev_priv->sb_lock); + dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy); I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control); + + DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n", + phy, dev_priv->chv_phy_control); + + assert_chv_phy_status(dev_priv); } static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, @@ -974,6 +1220,137 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control); vlv_set_power_well(dev_priv, power_well, false); + + DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n", + phy, dev_priv->chv_phy_control); + + /* PHY is fully reset now, so we can enable the PHY state asserts */ + dev_priv->chv_phy_assert[phy] = true; + + assert_chv_phy_status(dev_priv); +} + +static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy, + enum dpio_channel ch, bool override, unsigned int mask) +{ + enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C; + u32 reg, val, expected, actual; + + /* + * The BIOS can leave the PHY is some weird state + * where it doesn't fully power down some parts. + * Disable the asserts until the PHY has been fully + * reset (ie. the power well has been disabled at + * least once). + */ + if (!dev_priv->chv_phy_assert[phy]) + return; + + if (ch == DPIO_CH0) + reg = _CHV_CMN_DW0_CH0; + else + reg = _CHV_CMN_DW6_CH1; + + mutex_lock(&dev_priv->sb_lock); + val = vlv_dpio_read(dev_priv, pipe, reg); + mutex_unlock(&dev_priv->sb_lock); + + /* + * This assumes !override is only used when the port is disabled. + * All lanes should power down even without the override when + * the port is disabled. + */ + if (!override || mask == 0xf) { + expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN; + /* + * If CH1 common lane is not active anymore + * (eg. for pipe B DPLL) the entire channel will + * shut down, which causes the common lane registers + * to read as 0. That means we can't actually check + * the lane power down status bits, but as the entire + * register reads as 0 it's a good indication that the + * channel is indeed entirely powered down. + */ + if (ch == DPIO_CH1 && val == 0) + expected = 0; + } else if (mask != 0x0) { + expected = DPIO_ANYDL_POWERDOWN; + } else { + expected = 0; + } + + if (ch == DPIO_CH0) + actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0; + else + actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1; + actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN; + + WARN(actual != expected, + "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n", + !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN), + !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN), + reg, val); +} + +bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, + enum dpio_channel ch, bool override) +{ + struct i915_power_domains *power_domains = &dev_priv->power_domains; + bool was_override; + + mutex_lock(&power_domains->lock); + + was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + + if (override == was_override) + goto out; + + if (override) + dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + else + dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + + I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control); + + DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n", + phy, ch, dev_priv->chv_phy_control); + + assert_chv_phy_status(dev_priv); + +out: + mutex_unlock(&power_domains->lock); + + return was_override; +} + +void chv_phy_powergate_lanes(struct intel_encoder *encoder, + bool override, unsigned int mask) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct i915_power_domains *power_domains = &dev_priv->power_domains; + enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base)); + enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); + + mutex_lock(&power_domains->lock); + + dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch); + dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch); + + if (override) + dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + else + dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + + I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control); + + DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n", + phy, ch, mask, dev_priv->chv_phy_control); + + assert_chv_phy_status(dev_priv); + + assert_chv_phy_powergate(dev_priv, phy, ch, override, mask); + + mutex_unlock(&power_domains->lock); } static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, @@ -1042,53 +1419,29 @@ out: static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { + WARN_ON_ONCE(power_well->data != PIPE_A); + chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0); } static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A && - power_well->data != PIPE_B && - power_well->data != PIPE_C); + WARN_ON_ONCE(power_well->data != PIPE_A); chv_set_pipe_power_well(dev_priv, power_well, true); - if (power_well->data == PIPE_A) { - spin_lock_irq(&dev_priv->irq_lock); - valleyview_enable_display_irqs(dev_priv); - spin_unlock_irq(&dev_priv->irq_lock); - - /* - * During driver initialization/resume we can avoid restoring the - * part of the HW/SW state that will be inited anyway explicitly. - */ - if (dev_priv->power_domains.initializing) - return; - - intel_hpd_init(dev_priv); - - i915_redisable_vga_power_on(dev_priv->dev); - } + vlv_display_power_well_init(dev_priv); } static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A && - power_well->data != PIPE_B && - power_well->data != PIPE_C); + WARN_ON_ONCE(power_well->data != PIPE_A); - if (power_well->data == PIPE_A) { - spin_lock_irq(&dev_priv->irq_lock); - valleyview_disable_display_irqs(dev_priv); - spin_unlock_irq(&dev_priv->irq_lock); - } + vlv_display_power_well_deinit(dev_priv); chv_set_pipe_power_well(dev_priv, power_well, false); - - if (power_well->data == PIPE_A) - vlv_power_sequencer_reset(dev_priv); } /** @@ -1117,11 +1470,8 @@ void intel_display_power_get(struct drm_i915_private *dev_priv, mutex_lock(&power_domains->lock); for_each_power_well(i, power_well, BIT(domain), power_domains) { - if (!power_well->count++) { - DRM_DEBUG_KMS("enabling %s\n", power_well->name); - power_well->ops->enable(dev_priv, power_well); - power_well->hw_enabled = true; - } + if (!power_well->count++) + intel_power_well_enable(dev_priv, power_well); } power_domains->domain_use_count[domain]++; @@ -1149,17 +1499,18 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, mutex_lock(&power_domains->lock); - WARN_ON(!power_domains->domain_use_count[domain]); + WARN(!power_domains->domain_use_count[domain], + "Use count on domain %s is already zero\n", + intel_display_power_domain_str(domain)); power_domains->domain_use_count[domain]--; for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { - WARN_ON(!power_well->count); + WARN(!power_well->count, + "Use count on power well %s is already zero", + power_well->name); - if (!--power_well->count && i915.disable_power_well) { - DRM_DEBUG_KMS("disabling %s\n", power_well->name); - power_well->hw_enabled = false; - power_well->ops->disable(dev_priv, power_well); - } + if (!--power_well->count) + intel_power_well_disable(dev_priv, power_well); } mutex_unlock(&power_domains->lock); @@ -1167,25 +1518,20 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, intel_runtime_pm_put(dev_priv); } -#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) - #define HSW_ALWAYS_ON_POWER_DOMAINS ( \ BIT(POWER_DOMAIN_PIPE_A) | \ BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_LANES) | \ BIT(POWER_DOMAIN_PORT_CRT) | \ BIT(POWER_DOMAIN_PLLS) | \ BIT(POWER_DOMAIN_AUX_A) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_AUX_D) | \ + BIT(POWER_DOMAIN_GMBUS) | \ BIT(POWER_DOMAIN_INIT)) #define HSW_DISPLAY_POWER_DOMAINS ( \ (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \ @@ -1202,49 +1548,42 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, #define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK #define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT(POWER_DOMAIN_PORT_CRT) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_INIT)) #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_INIT)) #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_INIT)) #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_INIT)) #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_INIT)) #define CHV_DPIO_CMN_BC_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ BIT(POWER_DOMAIN_AUX_B) | \ BIT(POWER_DOMAIN_AUX_C) | \ BIT(POWER_DOMAIN_INIT)) #define CHV_DPIO_CMN_D_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ - BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_LANES) | \ BIT(POWER_DOMAIN_AUX_D) | \ BIT(POWER_DOMAIN_INIT)) @@ -1292,6 +1631,13 @@ static const struct i915_power_well_ops skl_power_well_ops = { .is_enabled = skl_power_well_enabled, }; +static const struct i915_power_well_ops gen9_dc_off_power_well_ops = { + .sync_hw = gen9_dc_off_power_well_sync_hw, + .enable = gen9_dc_off_power_well_enable, + .disable = gen9_dc_off_power_well_disable, + .is_enabled = gen9_dc_off_power_well_enabled, +}; + static struct i915_power_well hsw_power_wells[] = { { .name = "always-on", @@ -1347,6 +1693,7 @@ static struct i915_power_well vlv_power_wells[] = { .always_on = 1, .domains = VLV_ALWAYS_ON_POWER_DOMAINS, .ops = &i9xx_always_on_power_well_ops, + .data = PUNIT_POWER_WELL_ALWAYS_ON, }, { .name = "display", @@ -1430,21 +1777,6 @@ static struct i915_power_well chv_power_wells[] = { }, }; -static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv, - int power_well_id) -{ - struct i915_power_domains *power_domains = &dev_priv->power_domains; - struct i915_power_well *power_well; - int i; - - for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { - if (power_well->data == power_well_id) - return power_well; - } - - return NULL; -} - bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, int power_well_id) { @@ -1463,20 +1795,29 @@ static struct i915_power_well skl_power_wells[] = { .always_on = 1, .domains = SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS, .ops = &i9xx_always_on_power_well_ops, + .data = SKL_DISP_PW_ALWAYS_ON, }, { .name = "power well 1", - .domains = SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS, + /* Handled by the DMC firmware */ + .domains = 0, .ops = &skl_power_well_ops, .data = SKL_DISP_PW_1, }, { .name = "MISC IO power well", - .domains = SKL_DISPLAY_MISC_IO_POWER_DOMAINS, + /* Handled by the DMC firmware */ + .domains = 0, .ops = &skl_power_well_ops, .data = SKL_DISP_PW_MISC_IO, }, { + .name = "DC off", + .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS, + .ops = &gen9_dc_off_power_well_ops, + .data = SKL_DISP_PW_DC_OFF, + }, + { .name = "power well 2", .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS, .ops = &skl_power_well_ops, @@ -1508,6 +1849,34 @@ static struct i915_power_well skl_power_wells[] = { }, }; +void skl_pw1_misc_io_init(struct drm_i915_private *dev_priv) +{ + struct i915_power_well *well; + + if (!IS_SKYLAKE(dev_priv)) + return; + + well = lookup_power_well(dev_priv, SKL_DISP_PW_1); + intel_power_well_enable(dev_priv, well); + + well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO); + intel_power_well_enable(dev_priv, well); +} + +void skl_pw1_misc_io_fini(struct drm_i915_private *dev_priv) +{ + struct i915_power_well *well; + + if (!IS_SKYLAKE(dev_priv)) + return; + + well = lookup_power_well(dev_priv, SKL_DISP_PW_1); + intel_power_well_disable(dev_priv, well); + + well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO); + intel_power_well_disable(dev_priv, well); +} + static struct i915_power_well bxt_power_wells[] = { { .name = "always-on", @@ -1522,13 +1891,34 @@ static struct i915_power_well bxt_power_wells[] = { .data = SKL_DISP_PW_1, }, { + .name = "DC off", + .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS, + .ops = &gen9_dc_off_power_well_ops, + .data = SKL_DISP_PW_DC_OFF, + }, + { .name = "power well 2", .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS, .ops = &skl_power_well_ops, .data = SKL_DISP_PW_2, - } + }, }; +static int +sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv, + int disable_power_well) +{ + if (disable_power_well >= 0) + return !!disable_power_well; + + if (IS_BROXTON(dev_priv)) { + DRM_DEBUG_KMS("Disabling display power well support\n"); + return 0; + } + + return 1; +} + #define set_power_wells(power_domains, __power_wells) ({ \ (power_domains)->power_wells = (__power_wells); \ (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \ @@ -1545,6 +1935,11 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) { struct i915_power_domains *power_domains = &dev_priv->power_domains; + i915.disable_power_well = sanitize_disable_power_well_option(dev_priv, + i915.disable_power_well); + + BUILD_BUG_ON(POWER_DOMAIN_NUM > 31); + mutex_init(&power_domains->lock); /* @@ -1555,7 +1950,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) set_power_wells(power_domains, hsw_power_wells); } else if (IS_BROADWELL(dev_priv->dev)) { set_power_wells(power_domains, bdw_power_wells); - } else if (IS_SKYLAKE(dev_priv->dev)) { + } else if (IS_SKYLAKE(dev_priv->dev) || IS_KABYLAKE(dev_priv->dev)) { set_power_wells(power_domains, skl_power_wells); } else if (IS_BROXTON(dev_priv->dev)) { set_power_wells(power_domains, bxt_power_wells); @@ -1570,22 +1965,6 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) return 0; } -static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct device *device = &dev->pdev->dev; - - if (!HAS_RUNTIME_PM(dev)) - return; - - if (!intel_enable_rc6(dev)) - return; - - /* Make sure we're not suspended first. */ - pm_runtime_get_sync(device); - pm_runtime_disable(device); -} - /** * intel_power_domains_fini - finalizes the power domain structures * @dev_priv: i915 device instance @@ -1596,15 +1975,17 @@ static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv) */ void intel_power_domains_fini(struct drm_i915_private *dev_priv) { - intel_runtime_pm_disable(dev_priv); - /* The i915.ko module is still not prepared to be loaded when * the power well is not enabled, so just enable it in case * we're going to unload/reload. */ intel_display_set_init_power(dev_priv, true); + + /* Remove the refcount we took to keep power well support disabled. */ + if (!i915.disable_power_well) + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); } -static void intel_power_domains_resume(struct drm_i915_private *dev_priv) +static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv) { struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *power_well; @@ -1619,6 +2000,47 @@ static void intel_power_domains_resume(struct drm_i915_private *dev_priv) mutex_unlock(&power_domains->lock); } +static void skl_display_core_init(struct drm_i915_private *dev_priv, + bool resume) +{ + struct i915_power_domains *power_domains = &dev_priv->power_domains; + uint32_t val; + + gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); + + /* enable PCH reset handshake */ + val = I915_READ(HSW_NDE_RSTWRN_OPT); + I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE); + + /* enable PG1 and Misc I/O */ + mutex_lock(&power_domains->lock); + skl_pw1_misc_io_init(dev_priv); + mutex_unlock(&power_domains->lock); + + if (!resume) + return; + + skl_init_cdclk(dev_priv); + + if (dev_priv->csr.dmc_payload) + intel_csr_load_program(dev_priv); +} + +static void skl_display_core_uninit(struct drm_i915_private *dev_priv) +{ + struct i915_power_domains *power_domains = &dev_priv->power_domains; + + gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); + + skl_uninit_cdclk(dev_priv); + + /* The spec doesn't call for removing the reset handshake flag */ + /* disable PG1 and Misc I/O */ + mutex_lock(&power_domains->lock); + skl_pw1_misc_io_fini(dev_priv); + mutex_unlock(&power_domains->lock); +} + static void chv_phy_control_init(struct drm_i915_private *dev_priv) { struct i915_power_well *cmn_bc = @@ -1630,19 +2052,80 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv) * DISPLAY_PHY_CONTROL can get corrupted if read. As a * workaround never ever read DISPLAY_PHY_CONTROL, and * instead maintain a shadow copy ourselves. Use the actual - * power well state to reconstruct the expected initial - * value. + * power well state and lane status to reconstruct the + * expected initial value. */ dev_priv->chv_phy_control = PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) | PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) | - PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY0, DPIO_CH0) | - PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY0, DPIO_CH1) | - PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY1, DPIO_CH0); - if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) + PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) | + PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) | + PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0); + + /* + * If all lanes are disabled we leave the override disabled + * with all power down bits cleared to match the state we + * would use after disabling the port. Otherwise enable the + * override and set the lane powerdown bits accding to the + * current lane status. + */ + if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) { + uint32_t status = I915_READ(DPLL(PIPE_A)); + unsigned int mask; + + mask = status & DPLL_PORTB_READY_MASK; + if (mask == 0xf) + mask = 0x0; + else + dev_priv->chv_phy_control |= + PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0); + + dev_priv->chv_phy_control |= + PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0); + + mask = (status & DPLL_PORTC_READY_MASK) >> 4; + if (mask == 0xf) + mask = 0x0; + else + dev_priv->chv_phy_control |= + PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1); + + dev_priv->chv_phy_control |= + PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1); + dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0); - if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) + + dev_priv->chv_phy_assert[DPIO_PHY0] = false; + } else { + dev_priv->chv_phy_assert[DPIO_PHY0] = true; + } + + if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) { + uint32_t status = I915_READ(DPIO_PHY_STATUS); + unsigned int mask; + + mask = status & DPLL_PORTD_READY_MASK; + + if (mask == 0xf) + mask = 0x0; + else + dev_priv->chv_phy_control |= + PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0); + + dev_priv->chv_phy_control |= + PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0); + dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1); + + dev_priv->chv_phy_assert[DPIO_PHY1] = false; + } else { + dev_priv->chv_phy_assert[DPIO_PHY1] = true; + } + + I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control); + + DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n", + dev_priv->chv_phy_control); } static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv) @@ -1680,15 +2163,19 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv) * This function initializes the hardware power domain state and enables all * power domains using intel_display_set_init_power(). */ -void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) +void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume) { struct drm_device *dev = dev_priv->dev; struct i915_power_domains *power_domains = &dev_priv->power_domains; power_domains->initializing = true; - if (IS_CHERRYVIEW(dev)) { + if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + skl_display_core_init(dev_priv, resume); + } else if (IS_CHERRYVIEW(dev)) { + mutex_lock(&power_domains->lock); chv_phy_control_init(dev_priv); + mutex_unlock(&power_domains->lock); } else if (IS_VALLEYVIEW(dev)) { mutex_lock(&power_domains->lock); vlv_cmnlane_wa(dev_priv); @@ -1697,38 +2184,31 @@ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) /* For now, we need the power well to be always enabled. */ intel_display_set_init_power(dev_priv, true); - intel_power_domains_resume(dev_priv); + /* Disable power support if the user asked so. */ + if (!i915.disable_power_well) + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + intel_power_domains_sync_hw(dev_priv); power_domains->initializing = false; } /** - * intel_aux_display_runtime_get - grab an auxiliary power domain reference + * intel_power_domains_suspend - suspend power domain state * @dev_priv: i915 device instance * - * This function grabs a power domain reference for the auxiliary power domain - * (for access to the GMBUS and DP AUX blocks) and ensures that it and all its - * parents are powered up. Therefore users should only grab a reference to the - * innermost power domain they need. - * - * Any power domain reference obtained by this function must have a symmetric - * call to intel_aux_display_runtime_put() to release the reference again. + * This function prepares the hardware power domain state before entering + * system suspend. It must be paired with intel_power_domains_init_hw(). */ -void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv) +void intel_power_domains_suspend(struct drm_i915_private *dev_priv) { - intel_runtime_pm_get(dev_priv); -} + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + skl_display_core_uninit(dev_priv); -/** - * intel_aux_display_runtime_put - release an auxiliary power domain reference - * @dev_priv: i915 device instance - * - * This function drops the auxiliary power domain reference obtained by - * intel_aux_display_runtime_get() and might power down the corresponding - * hardware block right away if this is the last reference. - */ -void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv) -{ - intel_runtime_pm_put(dev_priv); + /* + * Even if power well support was disabled we still want to disable + * power wells while we are system suspended. + */ + if (!i915.disable_power_well) + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); } /** @@ -1820,8 +2300,6 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv) if (!HAS_RUNTIME_PM(dev)) return; - pm_runtime_set_active(device); - /* * RPM depends on RC6 to save restore the GT HW context, so make RC6 a * requirement. diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index aa2fd751609c..2e1da060b0e1 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -53,7 +53,7 @@ #define IS_DIGITAL(c) (c->output_flag & (SDVO_TMDS_MASK | SDVO_LVDS_MASK)) -static const char *tv_format_names[] = { +static const char * const tv_format_names[] = { "NTSC_M" , "NTSC_J" , "NTSC_443", "PAL_B" , "PAL_D" , "PAL_G" , "PAL_H" , "PAL_I" , "PAL_M" , @@ -63,7 +63,7 @@ static const char *tv_format_names[] = { "SECAM_60" }; -#define TV_FORMAT_NUM (sizeof(tv_format_names) / sizeof(*tv_format_names)) +#define TV_FORMAT_NUM ARRAY_SIZE(tv_format_names) struct intel_sdvo { struct intel_encoder base; @@ -74,7 +74,7 @@ struct intel_sdvo { struct i2c_adapter ddc; /* Register for the SDVO device: SDVOB or SDVOC */ - uint32_t sdvo_reg; + i915_reg_t sdvo_reg; /* Active outputs controlled by this SDVO output */ uint16_t controlled_output; @@ -107,6 +107,11 @@ struct intel_sdvo { bool color_range_auto; /** + * HDMI user specified aspect ratio + */ + enum hdmi_picture_aspect aspect_ratio; + + /** * This is set if we're going to treat the device as TV-out. * * While we have these nice friendly flags for output types that ought @@ -115,8 +120,7 @@ struct intel_sdvo { */ bool is_tv; - /* On different gens SDVOB is at different places. */ - bool is_sdvob; + enum port port; /* This is for current tv format name */ int tv_format_index; @@ -240,7 +244,7 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val) u32 bval = val, cval = val; int i; - if (intel_sdvo->sdvo_reg == PCH_SDVOB) { + if (HAS_PCH_SPLIT(dev_priv)) { I915_WRITE(intel_sdvo->sdvo_reg, val); POSTING_READ(intel_sdvo->sdvo_reg); /* @@ -254,7 +258,7 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val) return; } - if (intel_sdvo->sdvo_reg == GEN3_SDVOB) + if (intel_sdvo->port == PORT_B) cval = I915_READ(GEN3_SDVOC); else bval = I915_READ(GEN3_SDVOB); @@ -417,7 +421,7 @@ static const struct _sdvo_cmd_name { SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_HBUF_DATA), }; -#define SDVO_NAME(svdo) ((svdo)->is_sdvob ? "SDVOB" : "SDVOC") +#define SDVO_NAME(svdo) ((svdo)->port == PORT_B ? "SDVOB" : "SDVOC") static void intel_sdvo_debug_write(struct intel_sdvo *intel_sdvo, u8 cmd, const void *args, int args_len) @@ -452,7 +456,7 @@ static void intel_sdvo_debug_write(struct intel_sdvo *intel_sdvo, u8 cmd, DRM_DEBUG_KMS("%s: W: %02X %s\n", SDVO_NAME(intel_sdvo), cmd, buffer); } -static const char *cmd_status_names[] = { +static const char * const cmd_status_names[] = { "Power on", "Success", "Not supported", @@ -603,11 +607,11 @@ log_fail: return false; } -static int intel_sdvo_get_pixel_multiplier(struct drm_display_mode *mode) +static int intel_sdvo_get_pixel_multiplier(const struct drm_display_mode *adjusted_mode) { - if (mode->clock >= 100000) + if (adjusted_mode->crtc_clock >= 100000) return 1; - else if (mode->clock >= 50000) + else if (adjusted_mode->crtc_clock >= 50000) return 2; else return 4; @@ -1181,6 +1185,10 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, if (intel_sdvo->is_tv) i9xx_adjust_sdvo_tv_clock(pipe_config); + /* Set user selected PAR to incoming mode's member */ + if (intel_sdvo->is_hdmi) + adjusted_mode->picture_aspect_ratio = intel_sdvo->aspect_ratio; + return true; } @@ -1189,8 +1197,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder) struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc = to_intel_crtc(intel_encoder->base.crtc); - struct drm_display_mode *adjusted_mode = - &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; struct drm_display_mode *mode = &crtc->config->base.mode; struct intel_sdvo *intel_sdvo = to_sdvo(intel_encoder); u32 sdvox; @@ -1274,14 +1281,10 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder) sdvox |= SDVO_BORDER_ENABLE; } else { sdvox = I915_READ(intel_sdvo->sdvo_reg); - switch (intel_sdvo->sdvo_reg) { - case GEN3_SDVOB: + if (intel_sdvo->port == PORT_B) sdvox &= SDVOB_PRESERVE_MASK; - break; - case GEN3_SDVOC: + else sdvox &= SDVOC_PRESERVE_MASK; - break; - } sdvox |= (9 << 19) | SDVO_BORDER_ENABLE; } @@ -1456,12 +1459,23 @@ static void intel_disable_sdvo(struct intel_encoder *encoder) * matching DP port to be enabled on transcoder A. */ if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B) { + /* + * We get CPU/PCH FIFO underruns on the other pipe when + * doing the workaround. Sweep them under the rug. + */ + intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, false); + intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); + temp &= ~SDVO_PIPE_B_SELECT; temp |= SDVO_ENABLE; intel_sdvo_write_sdvox(intel_sdvo, temp); temp &= ~SDVO_ENABLE; intel_sdvo_write_sdvox(intel_sdvo, temp); + + intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A); + intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); + intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } } @@ -1508,51 +1522,6 @@ static void intel_enable_sdvo(struct intel_encoder *encoder) intel_sdvo_set_active_outputs(intel_sdvo, intel_sdvo->attached_output); } -/* Special dpms function to support cloning between dvo/sdvo/crt. */ -static void intel_sdvo_dpms(struct drm_connector *connector, int mode) -{ - struct drm_crtc *crtc; - struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector); - - /* dvo supports only 2 dpms states. */ - if (mode != DRM_MODE_DPMS_ON) - mode = DRM_MODE_DPMS_OFF; - - if (mode == connector->dpms) - return; - - connector->dpms = mode; - - /* Only need to change hw state when actually enabled */ - crtc = intel_sdvo->base.base.crtc; - if (!crtc) { - intel_sdvo->base.connectors_active = false; - return; - } - - /* We set active outputs manually below in case pipe dpms doesn't change - * due to cloning. */ - if (mode != DRM_MODE_DPMS_ON) { - intel_sdvo_set_active_outputs(intel_sdvo, 0); - if (0) - intel_sdvo_set_encoder_power_state(intel_sdvo, mode); - - intel_sdvo->base.connectors_active = false; - - intel_crtc_update_dpms(crtc); - } else { - intel_sdvo->base.connectors_active = true; - - intel_crtc_update_dpms(crtc); - - if (0) - intel_sdvo_set_encoder_power_state(intel_sdvo, mode); - intel_sdvo_set_active_outputs(intel_sdvo, intel_sdvo->attached_output); - } - - intel_modeset_check_state(connector->dev); -} - static enum drm_mode_status intel_sdvo_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) @@ -2089,6 +2058,23 @@ intel_sdvo_set_property(struct drm_connector *connector, goto done; } + if (property == connector->dev->mode_config.aspect_ratio_property) { + switch (val) { + case DRM_MODE_PICTURE_ASPECT_NONE: + intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_NONE; + break; + case DRM_MODE_PICTURE_ASPECT_4_3: + intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_4_3; + break; + case DRM_MODE_PICTURE_ASPECT_16_9: + intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_16_9; + break; + default: + return -EINVAL; + } + goto done; + } + #define CHECK_PROPERTY(name, NAME) \ if (intel_sdvo_connector->name == property) { \ if (intel_sdvo_connector->cur_##name == temp_value) return 0; \ @@ -2190,7 +2176,7 @@ done: } static const struct drm_connector_funcs intel_sdvo_connector_funcs = { - .dpms = intel_sdvo_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_sdvo_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_sdvo_set_property, @@ -2267,11 +2253,11 @@ intel_sdvo_guess_ddc_bus(struct intel_sdvo *sdvo) */ static void intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv, - struct intel_sdvo *sdvo, u32 reg) + struct intel_sdvo *sdvo) { struct sdvo_device_mapping *mapping; - if (sdvo->is_sdvob) + if (sdvo->port == PORT_B) mapping = &(dev_priv->sdvo_mappings[0]); else mapping = &(dev_priv->sdvo_mappings[1]); @@ -2284,12 +2270,12 @@ intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv, static void intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv, - struct intel_sdvo *sdvo, u32 reg) + struct intel_sdvo *sdvo) { struct sdvo_device_mapping *mapping; u8 pin; - if (sdvo->is_sdvob) + if (sdvo->port == PORT_B) mapping = &dev_priv->sdvo_mappings[0]; else mapping = &dev_priv->sdvo_mappings[1]; @@ -2327,7 +2313,7 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo) struct drm_i915_private *dev_priv = dev->dev_private; struct sdvo_device_mapping *my_mapping, *other_mapping; - if (sdvo->is_sdvob) { + if (sdvo->port == PORT_B) { my_mapping = &dev_priv->sdvo_mappings[0]; other_mapping = &dev_priv->sdvo_mappings[1]; } else { @@ -2352,7 +2338,7 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo) /* No SDVO device info is found for another DVO port, * so use mapping assumption we had before BIOS parsing. */ - if (sdvo->is_sdvob) + if (sdvo->port == PORT_B) return 0x70; else return 0x72; @@ -2428,6 +2414,8 @@ intel_sdvo_add_hdmi_properties(struct intel_sdvo *intel_sdvo, intel_attach_broadcast_rgb_property(&connector->base.base); intel_sdvo->color_range_auto = true; } + intel_attach_aspect_ratio_property(&connector->base.base); + intel_sdvo->aspect_ratio = HDMI_PICTURE_ASPECT_NONE; } static struct intel_sdvo_connector *intel_sdvo_connector_alloc(void) @@ -2957,27 +2945,41 @@ intel_sdvo_init_ddc_proxy(struct intel_sdvo *sdvo, return i2c_add_adapter(&sdvo->ddc) == 0; } -bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob) +static void assert_sdvo_port_valid(const struct drm_i915_private *dev_priv, + enum port port) +{ + if (HAS_PCH_SPLIT(dev_priv)) + WARN_ON(port != PORT_B); + else + WARN_ON(port != PORT_B && port != PORT_C); +} + +bool intel_sdvo_init(struct drm_device *dev, + i915_reg_t sdvo_reg, enum port port) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_encoder *intel_encoder; struct intel_sdvo *intel_sdvo; int i; + + assert_sdvo_port_valid(dev_priv, port); + intel_sdvo = kzalloc(sizeof(*intel_sdvo), GFP_KERNEL); if (!intel_sdvo) return false; intel_sdvo->sdvo_reg = sdvo_reg; - intel_sdvo->is_sdvob = is_sdvob; + intel_sdvo->port = port; intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, intel_sdvo) >> 1; - intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo, sdvo_reg); + intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo); if (!intel_sdvo_init_ddc_proxy(intel_sdvo, dev)) goto err_i2c_bus; /* encoder type will be decided later */ intel_encoder = &intel_sdvo->base; intel_encoder->type = INTEL_OUTPUT_SDVO; - drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0); + drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0, + NULL); /* Read the regs to test if we can talk to the device */ for (i = 0; i < 0x40; i++) { @@ -3018,8 +3020,10 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob) * hotplug lines. */ if (intel_sdvo->hotplug_active) { - intel_encoder->hpd_pin = - intel_sdvo->is_sdvob ? HPD_SDVO_B : HPD_SDVO_C; + if (intel_sdvo->port == PORT_B) + intel_encoder->hpd_pin = HPD_SDVO_B; + else + intel_encoder->hpd_pin = HPD_SDVO_C; } /* @@ -3032,7 +3036,7 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob) */ intel_sdvo->base.cloneable = 0; - intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg); + intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo); /* Set the input timing to the screen. Assume always input 0. */ if (!intel_sdvo_set_target_input(intel_sdvo)) diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index 8193a35388d7..dbf421351b5c 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -53,13 +53,15 @@ format_is_yuv(uint32_t format) } } -static int usecs_to_scanlines(const struct drm_display_mode *mode, int usecs) +static int usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, + int usecs) { /* paranoia */ - if (!mode->crtc_htotal) + if (!adjusted_mode->crtc_htotal) return 1; - return DIV_ROUND_UP(usecs * mode->crtc_clock, 1000 * mode->crtc_htotal); + return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock, + 1000 * adjusted_mode->crtc_htotal); } /** @@ -75,36 +77,36 @@ static int usecs_to_scanlines(const struct drm_display_mode *mode, int usecs) * until a subsequent call to intel_pipe_update_end(). That is done to * avoid random delays. The value written to @start_vbl_count should be * supplied to intel_pipe_update_end() for error checking. - * - * Return: true if the call was successful */ -bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl_count) +void intel_pipe_update_start(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; enum pipe pipe = crtc->pipe; long timeout = msecs_to_jiffies_timeout(1); int scanline, min, max, vblank_start; wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base); DEFINE_WAIT(wait); - vblank_start = mode->crtc_vblank_start; - if (mode->flags & DRM_MODE_FLAG_INTERLACE) + vblank_start = adjusted_mode->crtc_vblank_start; + if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) vblank_start = DIV_ROUND_UP(vblank_start, 2); /* FIXME needs to be calibrated sensibly */ - min = vblank_start - usecs_to_scanlines(mode, 100); + min = vblank_start - usecs_to_scanlines(adjusted_mode, 100); max = vblank_start - 1; + local_irq_disable(); + if (min <= 0 || max <= 0) - return false; + return; if (WARN_ON(drm_crtc_vblank_get(&crtc->base))) - return false; - - local_irq_disable(); + return; - trace_i915_pipe_update_start(crtc, min, max); + crtc->debug.min_vbl = min; + crtc->debug.max_vbl = max; + trace_i915_pipe_update_start(crtc); for (;;) { /* @@ -135,11 +137,12 @@ bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl_count) drm_crtc_vblank_put(&crtc->base); - *start_vbl_count = dev->driver->get_vblank_counter(dev, pipe); - - trace_i915_pipe_update_vblank_evaded(crtc, min, max, *start_vbl_count); + crtc->debug.scanline_start = scanline; + crtc->debug.start_vbl_time = ktime_get(); + crtc->debug.start_vbl_count = + dev->driver->get_vblank_counter(dev, pipe); - return true; + trace_i915_pipe_update_vblank_evaded(crtc); } /** @@ -151,19 +154,27 @@ bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl_count) * re-enables interrupts and verifies the update was actually completed * before a vblank using the value of @start_vbl_count. */ -void intel_pipe_update_end(struct intel_crtc *crtc, u32 start_vbl_count) +void intel_pipe_update_end(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; enum pipe pipe = crtc->pipe; + int scanline_end = intel_get_crtc_scanline(crtc); u32 end_vbl_count = dev->driver->get_vblank_counter(dev, pipe); + ktime_t end_vbl_time = ktime_get(); - trace_i915_pipe_update_end(crtc, end_vbl_count); + trace_i915_pipe_update_end(crtc, end_vbl_count, scanline_end); local_irq_enable(); - if (start_vbl_count != end_vbl_count) - DRM_ERROR("Atomic update failure on pipe %c (start=%u end=%u)\n", - pipe_name(pipe), start_vbl_count, end_vbl_count); + if (crtc->debug.start_vbl_count && + crtc->debug.start_vbl_count != end_vbl_count) { + DRM_ERROR("Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n", + pipe_name(pipe), crtc->debug.start_vbl_count, + end_vbl_count, + ktime_us_delta(end_vbl_time, crtc->debug.start_vbl_time), + crtc->debug.min_vbl, crtc->debug.max_vbl, + crtc->debug.scanline_start, scanline_end); + } } static void @@ -181,9 +192,9 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc, const int pipe = intel_plane->pipe; const int plane = intel_plane->plane + 1; u32 plane_ctl, stride_div, stride; - int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); - const struct drm_intel_sprite_colorkey *key = &intel_plane->ckey; - unsigned long surf_addr; + const struct drm_intel_sprite_colorkey *key = + &to_intel_plane_state(drm_plane->state)->ckey; + u32 surf_addr; u32 tile_height, plane_offset, plane_size; unsigned int rotation; int x_offset, y_offset; @@ -191,6 +202,7 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc, int scaler_id; plane_ctl = PLANE_CTL_ENABLE | + PLANE_CTL_PIPE_GAMMA_ENABLE | PLANE_CTL_PIPE_CSC_ENABLE; plane_ctl |= skl_plane_ctl_format(fb->pixel_format); @@ -199,10 +211,6 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc, rotation = drm_plane->state->rotation; plane_ctl |= skl_plane_ctl_rotation(rotation); - intel_update_sprite_watermarks(drm_plane, crtc, src_w, src_h, - pixel_size, true, - src_w != crtc_w || src_h != crtc_h); - stride_div = intel_fb_stride_alignment(dev, fb->modifier[0], fb->pixel_format); @@ -225,12 +233,12 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc, else if (key->flags & I915_SET_COLORKEY_SOURCE) plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE; - surf_addr = intel_plane_obj_offset(intel_plane, obj); + surf_addr = intel_plane_obj_offset(intel_plane, obj, 0); if (intel_rotation_90_or_270(rotation)) { /* stride: Surface height in tiles */ tile_height = intel_tile_height(dev, fb->pixel_format, - fb->modifier[0]); + fb->modifier[0], 0); stride = DIV_ROUND_UP(fb->height, tile_height); plane_size = (src_w << 16) | src_h; x_offset = stride * tile_height - y - (src_h + 1); @@ -272,7 +280,7 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc, } static void -skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc, bool force) +skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) { struct drm_device *dev = dplane->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -284,8 +292,6 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc, bool force) I915_WRITE(PLANE_SURF(pipe, plane), 0); POSTING_READ(PLANE_SURF(pipe, plane)); - - intel_update_sprite_watermarks(dplane, crtc, 0, 0, 0, false, false); } static void @@ -344,7 +350,8 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, u32 sprctl; unsigned long sprsurf_offset, linear_offset; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); - const struct drm_intel_sprite_colorkey *key = &intel_plane->ckey; + const struct drm_intel_sprite_colorkey *key = + &to_intel_plane_state(dplane->state)->ckey; sprctl = SP_ENABLE; @@ -400,10 +407,6 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, if (obj->tiling_mode != I915_TILING_NONE) sprctl |= SP_TILED; - intel_update_sprite_watermarks(dplane, crtc, src_w, src_h, - pixel_size, true, - src_w != crtc_w || src_h != crtc_h); - /* Sizes are 0 based */ src_w--; src_h--; @@ -411,7 +414,8 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, crtc_h--; linear_offset = y * fb->pitches[0] + x * pixel_size; - sprsurf_offset = intel_gen4_compute_page_offset(&x, &y, + sprsurf_offset = intel_gen4_compute_page_offset(dev_priv, + &x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); @@ -455,7 +459,7 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, } static void -vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc, bool force) +vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) { struct drm_device *dev = dplane->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -467,8 +471,6 @@ vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc, bool force) I915_WRITE(SPSURF(pipe, plane), 0); POSTING_READ(SPSURF(pipe, plane)); - - intel_update_sprite_watermarks(dplane, crtc, 0, 0, 0, false, false); } static void @@ -487,7 +489,8 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, u32 sprctl, sprscale = 0; unsigned long sprsurf_offset, linear_offset; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); - const struct drm_intel_sprite_colorkey *key = &intel_plane->ckey; + const struct drm_intel_sprite_colorkey *key = + &to_intel_plane_state(plane->state)->ckey; sprctl = SPRITE_ENABLE; @@ -531,10 +534,6 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, if (IS_HASWELL(dev) || IS_BROADWELL(dev)) sprctl |= SPRITE_PIPE_CSC_ENABLE; - intel_update_sprite_watermarks(plane, crtc, src_w, src_h, pixel_size, - true, - src_w != crtc_w || src_h != crtc_h); - /* Sizes are 0 based */ src_w--; src_h--; @@ -546,7 +545,8 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, linear_offset = y * fb->pitches[0] + x * pixel_size; sprsurf_offset = - intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, + intel_gen4_compute_page_offset(dev_priv, + &x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); linear_offset -= sprsurf_offset; @@ -595,14 +595,14 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, } static void -ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc, bool force) +ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) { struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(plane); int pipe = intel_plane->pipe; - I915_WRITE(SPRCTL(pipe), I915_READ(SPRCTL(pipe)) & ~SPRITE_ENABLE); + I915_WRITE(SPRCTL(pipe), 0); /* Can't leave the scaler enabled... */ if (intel_plane->can_scale) I915_WRITE(SPRSCALE(pipe), 0); @@ -627,7 +627,8 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, unsigned long dvssurf_offset, linear_offset; u32 dvscntr, dvsscale; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); - const struct drm_intel_sprite_colorkey *key = &intel_plane->ckey; + const struct drm_intel_sprite_colorkey *key = + &to_intel_plane_state(plane->state)->ckey; dvscntr = DVS_ENABLE; @@ -666,10 +667,6 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, if (IS_GEN6(dev)) dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */ - intel_update_sprite_watermarks(plane, crtc, src_w, src_h, - pixel_size, true, - src_w != crtc_w || src_h != crtc_h); - /* Sizes are 0 based */ src_w--; src_h--; @@ -682,7 +679,8 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, linear_offset = y * fb->pitches[0] + x * pixel_size; dvssurf_offset = - intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, + intel_gen4_compute_page_offset(dev_priv, + &x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); linear_offset -= dvssurf_offset; @@ -722,7 +720,7 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, } static void -ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc, bool force) +ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) { struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -739,11 +737,12 @@ ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc, bool force) static int intel_check_sprite_plane(struct drm_plane *plane, + struct intel_crtc_state *crtc_state, struct intel_plane_state *state) { struct drm_device *dev = plane->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc); - struct intel_crtc_state *crtc_state; + struct drm_crtc *crtc = state->base.crtc; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_framebuffer *fb = state->base.fb; int crtc_x, crtc_y; @@ -756,15 +755,10 @@ intel_check_sprite_plane(struct drm_plane *plane, int max_scale, min_scale; bool can_scale; int pixel_size; - int ret; - - intel_crtc = intel_crtc ? intel_crtc : to_intel_crtc(plane->crtc); - crtc_state = state->base.state ? - intel_atomic_get_crtc_state(state->base.state, intel_crtc) : NULL; if (!fb) { state->visible = false; - goto finish; + return 0; } /* Don't modify another pipe's plane */ @@ -782,7 +776,7 @@ intel_check_sprite_plane(struct drm_plane *plane, /* setup can_scale, min_scale, max_scale */ if (INTEL_INFO(dev)->gen >= 9) { /* use scaler when colorkey is not required */ - if (intel_plane->ckey.flags == I915_SET_COLORKEY_NONE) { + if (state->ckey.flags == I915_SET_COLORKEY_NONE) { can_scale = 1; min_scale = 1; max_scale = skl_max_scale(intel_crtc, crtc_state); @@ -802,7 +796,6 @@ intel_check_sprite_plane(struct drm_plane *plane, * coordinates and sizes. We probably need some way to decide whether * more strict checking should be done instead. */ - drm_rect_rotate(src, fb->width << 16, fb->height << 16, state->base.rotation); @@ -812,7 +805,7 @@ intel_check_sprite_plane(struct drm_plane *plane, vscale = drm_rect_calc_vscale_relaxed(src, dst, min_scale, max_scale); BUG_ON(vscale < 0); - state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale); + state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale); crtc_x = dst->x1; crtc_y = dst->y1; @@ -824,8 +817,8 @@ intel_check_sprite_plane(struct drm_plane *plane, hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale); if (hscale < 0) { DRM_DEBUG_KMS("Horizontal scaling factor out of limits\n"); - drm_rect_debug_print(src, true); - drm_rect_debug_print(dst, false); + drm_rect_debug_print("src: ", src, true); + drm_rect_debug_print("dst: ", dst, false); return hscale; } @@ -833,8 +826,8 @@ intel_check_sprite_plane(struct drm_plane *plane, vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale); if (vscale < 0) { DRM_DEBUG_KMS("Vertical scaling factor out of limits\n"); - drm_rect_debug_print(src, true); - drm_rect_debug_print(dst, false); + drm_rect_debug_print("src: ", src, true); + drm_rect_debug_print("dst: ", dst, false); return vscale; } @@ -917,36 +910,6 @@ intel_check_sprite_plane(struct drm_plane *plane, dst->y1 = crtc_y; dst->y2 = crtc_y + crtc_h; -finish: - /* - * If the sprite is completely covering the primary plane, - * we can disable the primary and save power. - */ - if (intel_crtc->active) { - intel_crtc->atomic.fb_bits |= - INTEL_FRONTBUFFER_SPRITE(intel_crtc->pipe); - - if (intel_wm_need_update(plane, &state->base)) - intel_crtc->atomic.update_wm = true; - - if (!state->visible) { - /* - * Avoid underruns when disabling the sprite. - * FIXME remove once watermark updates are done properly. - */ - intel_crtc->atomic.wait_vblank = true; - intel_crtc->atomic.update_sprite_watermarks |= - (1 << drm_plane_index(plane)); - } - } - - if (INTEL_INFO(dev)->gen >= 9) { - ret = skl_update_scaler_users(intel_crtc, crtc_state, intel_plane, - state, 0); - if (ret) - return ret; - } - return 0; } @@ -955,34 +918,22 @@ intel_commit_sprite_plane(struct drm_plane *plane, struct intel_plane_state *state) { struct drm_crtc *crtc = state->base.crtc; - struct intel_crtc *intel_crtc; struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_framebuffer *fb = state->base.fb; - int crtc_x, crtc_y; - unsigned int crtc_w, crtc_h; - uint32_t src_x, src_y, src_w, src_h; crtc = crtc ? crtc : plane->crtc; - intel_crtc = to_intel_crtc(crtc); - - plane->fb = fb; - - if (intel_crtc->active) { - if (state->visible) { - crtc_x = state->dst.x1; - crtc_y = state->dst.y1; - crtc_w = drm_rect_width(&state->dst); - crtc_h = drm_rect_height(&state->dst); - src_x = state->src.x1 >> 16; - src_y = state->src.y1 >> 16; - src_w = drm_rect_width(&state->src) >> 16; - src_h = drm_rect_height(&state->src) >> 16; - intel_plane->update_plane(plane, crtc, fb, - crtc_x, crtc_y, crtc_w, crtc_h, - src_x, src_y, src_w, src_h); - } else { - intel_plane->disable_plane(plane, crtc, false); - } + + if (state->visible) { + intel_plane->update_plane(plane, crtc, fb, + state->dst.x1, state->dst.y1, + drm_rect_width(&state->dst), + drm_rect_height(&state->dst), + state->src.x1 >> 16, + state->src.y1 >> 16, + drm_rect_width(&state->src) >> 16, + drm_rect_height(&state->src) >> 16); + } else { + intel_plane->disable_plane(plane, crtc); } } @@ -991,7 +942,9 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data, { struct drm_intel_sprite_colorkey *set = data; struct drm_plane *plane; - struct intel_plane *intel_plane; + struct drm_plane_state *plane_state; + struct drm_atomic_state *state; + struct drm_modeset_acquire_ctx ctx; int ret = 0; /* Make sure we don't try to enable both src & dest simultaneously */ @@ -1002,50 +955,41 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data, set->flags & I915_SET_COLORKEY_DESTINATION) return -EINVAL; - drm_modeset_lock_all(dev); - plane = drm_plane_find(dev, set->plane_id); - if (!plane || plane->type != DRM_PLANE_TYPE_OVERLAY) { - ret = -ENOENT; - goto out_unlock; - } + if (!plane || plane->type != DRM_PLANE_TYPE_OVERLAY) + return -ENOENT; - intel_plane = to_intel_plane(plane); + drm_modeset_acquire_init(&ctx, 0); - if (INTEL_INFO(dev)->gen >= 9) { - /* plane scaling and colorkey are mutually exclusive */ - if (to_intel_plane_state(plane->state)->scaler_id >= 0) { - DRM_ERROR("colorkey not allowed with scaler\n"); - ret = -EINVAL; - goto out_unlock; - } + state = drm_atomic_state_alloc(plane->dev); + if (!state) { + ret = -ENOMEM; + goto out; } + state->acquire_ctx = &ctx; + + while (1) { + plane_state = drm_atomic_get_plane_state(state, plane); + ret = PTR_ERR_OR_ZERO(plane_state); + if (!ret) { + to_intel_plane_state(plane_state)->ckey = *set; + ret = drm_atomic_commit(state); + } - intel_plane->ckey = *set; - - /* - * The only way this could fail would be due to - * the current plane state being unsupportable already, - * and we dont't consider that an error for the - * colorkey ioctl. So just ignore any error. - */ - intel_plane_restore(plane); + if (ret != -EDEADLK) + break; -out_unlock: - drm_modeset_unlock_all(dev); - return ret; -} + drm_atomic_state_clear(state); + drm_modeset_backoff(&ctx); + } -int intel_plane_restore(struct drm_plane *plane) -{ - if (!plane->crtc || !plane->state->fb) - return 0; + if (ret) + drm_atomic_state_free(state); - return drm_plane_helper_update(plane, plane->crtc, plane->state->fb, - plane->state->crtc_x, plane->state->crtc_y, - plane->state->crtc_w, plane->state->crtc_h, - plane->state->src_x, plane->state->src_y, - plane->state->src_w, plane->state->src_h); +out: + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + return ret; } static const uint32_t ilk_plane_formats[] = { @@ -1172,14 +1116,14 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) intel_plane->pipe = pipe; intel_plane->plane = plane; + intel_plane->frontbuffer_bit = INTEL_FRONTBUFFER_SPRITE(pipe, plane); intel_plane->check_plane = intel_check_sprite_plane; intel_plane->commit_plane = intel_commit_sprite_plane; - intel_plane->ckey.flags = I915_SET_COLORKEY_NONE; possible_crtcs = (1 << pipe); ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, &intel_plane_funcs, plane_formats, num_plane_formats, - DRM_PLANE_TYPE_OVERLAY); + DRM_PLANE_TYPE_OVERLAY, NULL); if (ret) { kfree(intel_plane); goto out; @@ -1189,6 +1133,6 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) drm_plane_helper_add(&intel_plane->base, &intel_plane_helper_funcs); - out: +out: return ret; } diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 8b9d325bda3c..948cbff6c62e 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -1138,13 +1138,13 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder) j = 0; for (i = 0; i < 60; i++) - I915_WRITE(TV_H_LUMA_0 + (i<<2), tv_mode->filter_table[j++]); + I915_WRITE(TV_H_LUMA(i), tv_mode->filter_table[j++]); for (i = 0; i < 60; i++) - I915_WRITE(TV_H_CHROMA_0 + (i<<2), tv_mode->filter_table[j++]); + I915_WRITE(TV_H_CHROMA(i), tv_mode->filter_table[j++]); for (i = 0; i < 43; i++) - I915_WRITE(TV_V_LUMA_0 + (i<<2), tv_mode->filter_table[j++]); + I915_WRITE(TV_V_LUMA(i), tv_mode->filter_table[j++]); for (i = 0; i < 43; i++) - I915_WRITE(TV_V_CHROMA_0 + (i<<2), tv_mode->filter_table[j++]); + I915_WRITE(TV_V_CHROMA(i), tv_mode->filter_table[j++]); I915_WRITE(TV_DAC, I915_READ(TV_DAC) & TV_DAC_SAVE); I915_WRITE(TV_CTL, tv_ctl); } @@ -1291,7 +1291,7 @@ static void intel_tv_find_better_format(struct drm_connector *connector) return; - for (i = 0; i < sizeof(tv_modes) / sizeof(*tv_modes); i++) { + for (i = 0; i < ARRAY_SIZE(tv_modes); i++) { tv_mode = tv_modes + i; if ((intel_tv->type == DRM_MODE_CONNECTOR_Component) == @@ -1509,7 +1509,7 @@ out: } static const struct drm_connector_funcs intel_tv_connector_funcs = { - .dpms = intel_connector_dpms, + .dpms = drm_atomic_helper_connector_dpms, .detect = intel_tv_detect, .destroy = intel_tv_destroy, .set_property = intel_tv_set_property, @@ -1579,7 +1579,7 @@ intel_tv_init(struct drm_device *dev) struct intel_encoder *intel_encoder; struct intel_connector *intel_connector; u32 tv_dac_on, tv_dac_off, save_tv_dac; - char *tv_format_names[ARRAY_SIZE(tv_modes)]; + const char *tv_format_names[ARRAY_SIZE(tv_modes)]; int i, initial_mode = 0; if ((I915_READ(TV_CTL) & TV_FUSE_STATE_MASK) == TV_FUSE_STATE_DISABLED) @@ -1645,7 +1645,7 @@ intel_tv_init(struct drm_device *dev) DRM_MODE_CONNECTOR_SVIDEO); drm_encoder_init(dev, &intel_encoder->base, &intel_tv_enc_funcs, - DRM_MODE_ENCODER_TVDAC); + DRM_MODE_ENCODER_TVDAC, NULL); intel_encoder->compute_config = intel_tv_compute_config; intel_encoder->get_config = intel_tv_get_config; @@ -1677,7 +1677,7 @@ intel_tv_init(struct drm_device *dev) /* Create TV properties then attach current values */ for (i = 0; i < ARRAY_SIZE(tv_modes); i++) - tv_format_names[i] = (char *)tv_modes[i].name; + tv_format_names[i] = tv_modes[i].name; drm_mode_create_tv_properties(dev, ARRAY_SIZE(tv_modes), tv_format_names); diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 260389acfb77..c2358ba78b30 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -27,21 +27,9 @@ #include <linux/pm_runtime.h> -#define FORCEWAKE_ACK_TIMEOUT_MS 2 +#define FORCEWAKE_ACK_TIMEOUT_MS 50 -#define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__)) -#define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__)) - -#define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__)) -#define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__)) - -#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__)) -#define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__)) - -#define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__)) -#define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__)) - -#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__) +#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__)) static const char * const forcewake_domain_names[] = { "render", @@ -52,8 +40,7 @@ static const char * const forcewake_domain_names[] = { const char * intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id) { - BUILD_BUG_ON((sizeof(forcewake_domain_names)/sizeof(const char *)) != - FW_DOMAIN_ID_COUNT); + BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT); if (id >= 0 && id < FW_DOMAIN_ID_COUNT) return forcewake_domain_names[id]; @@ -73,7 +60,7 @@ assert_device_not_suspended(struct drm_i915_private *dev_priv) static inline void fw_domain_reset(const struct intel_uncore_forcewake_domain *d) { - WARN_ON(d->reg_set == 0); + WARN_ON(!i915_mmio_reg_valid(d->reg_set)); __raw_i915_write32(d->i915, d->reg_set, d->val_reset); } @@ -119,7 +106,7 @@ static inline void fw_domain_posting_read(const struct intel_uncore_forcewake_domain *d) { /* something from same cacheline, but not from the set register */ - if (d->reg_post) + if (i915_mmio_reg_valid(d->reg_post)) __raw_posting_read(d->i915, d->reg_post); } @@ -526,8 +513,7 @@ void assert_forcewakes_inactive(struct drm_i915_private *dev_priv) } /* We give fast paths for the really cool registers */ -#define NEEDS_FORCE_WAKE(dev_priv, reg) \ - ((reg) < 0x40000 && (reg) != FORCEWAKE) +#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000) #define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end)) @@ -590,7 +576,7 @@ void assert_forcewakes_inactive(struct drm_i915_private *dev_priv) REG_RANGE((reg), 0x9400, 0x9800) #define FORCEWAKE_GEN9_BLITTER_RANGE_OFFSET(reg) \ - ((reg) < 0x40000 &&\ + ((reg) < 0x40000 && \ !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg) && \ !FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg) && \ !FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg) && \ @@ -606,8 +592,8 @@ ilk_dummy_write(struct drm_i915_private *dev_priv) } static void -hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read, - bool before) +hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, + i915_reg_t reg, bool read, bool before) { const char *op = read ? "reading" : "writing to"; const char *when = before ? "before" : "after"; @@ -617,7 +603,7 @@ hsw_unclaimed_reg_debug(struct drm_i915_private *dev_priv, u32 reg, bool read, if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { WARN(1, "Unclaimed register detected %s %s register 0x%x\n", - when, op, reg); + when, op, i915_mmio_reg_offset(reg)); __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); i915.mmio_debug--; /* Only report the first N failures */ } @@ -650,7 +636,7 @@ hsw_unclaimed_reg_detect(struct drm_i915_private *dev_priv) #define __gen2_read(x) \ static u##x \ -gen2_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ +gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ GEN2_READ_HEADER(x); \ val = __raw_i915_read##x(dev_priv, reg); \ GEN2_READ_FOOTER; \ @@ -658,7 +644,7 @@ gen2_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ #define __gen5_read(x) \ static u##x \ -gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ +gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ GEN2_READ_HEADER(x); \ ilk_dummy_write(dev_priv); \ val = __raw_i915_read##x(dev_priv, reg); \ @@ -681,6 +667,7 @@ __gen2_read(64) #undef GEN2_READ_HEADER #define GEN6_READ_HEADER(x) \ + u32 offset = i915_mmio_reg_offset(reg); \ unsigned long irqflags; \ u##x val = 0; \ assert_device_not_suspended(dev_priv); \ @@ -715,20 +702,12 @@ static inline void __force_wake_get(struct drm_i915_private *dev_priv, dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains); } -#define __vgpu_read(x) \ -static u##x \ -vgpu_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ - GEN6_READ_HEADER(x); \ - val = __raw_i915_read##x(dev_priv, reg); \ - GEN6_READ_FOOTER; \ -} - #define __gen6_read(x) \ static u##x \ -gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ +gen6_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ GEN6_READ_HEADER(x); \ hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \ - if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \ + if (NEEDS_FORCE_WAKE(offset)) \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ val = __raw_i915_read##x(dev_priv, reg); \ hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \ @@ -737,59 +716,66 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ #define __vlv_read(x) \ static u##x \ -vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ +vlv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ + enum forcewake_domains fw_engine = 0; \ GEN6_READ_HEADER(x); \ - if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ - else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \ + if (!NEEDS_FORCE_WAKE(offset)) \ + fw_engine = 0; \ + else if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_RENDER; \ + else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_MEDIA; \ + if (fw_engine) \ + __force_wake_get(dev_priv, fw_engine); \ val = __raw_i915_read##x(dev_priv, reg); \ GEN6_READ_FOOTER; \ } #define __chv_read(x) \ static u##x \ -chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ +chv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ + enum forcewake_domains fw_engine = 0; \ GEN6_READ_HEADER(x); \ - if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ - else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \ - else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, \ - FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \ + if (!NEEDS_FORCE_WAKE(offset)) \ + fw_engine = 0; \ + else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_RENDER; \ + else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_MEDIA; \ + else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \ + if (fw_engine) \ + __force_wake_get(dev_priv, fw_engine); \ val = __raw_i915_read##x(dev_priv, reg); \ GEN6_READ_FOOTER; \ } -#define SKL_NEEDS_FORCE_WAKE(dev_priv, reg) \ - ((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg)) +#define SKL_NEEDS_FORCE_WAKE(reg) \ + ((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg)) #define __gen9_read(x) \ static u##x \ -gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ +gen9_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ enum forcewake_domains fw_engine; \ GEN6_READ_HEADER(x); \ - if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) \ + hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \ + if (!SKL_NEEDS_FORCE_WAKE(offset)) \ fw_engine = 0; \ - else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \ + else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \ fw_engine = FORCEWAKE_RENDER; \ - else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \ + else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \ fw_engine = FORCEWAKE_MEDIA; \ - else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \ + else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \ fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \ else \ fw_engine = FORCEWAKE_BLITTER; \ if (fw_engine) \ __force_wake_get(dev_priv, fw_engine); \ val = __raw_i915_read##x(dev_priv, reg); \ + hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \ GEN6_READ_FOOTER; \ } -__vgpu_read(8) -__vgpu_read(16) -__vgpu_read(32) -__vgpu_read(64) __gen9_read(8) __gen9_read(16) __gen9_read(32) @@ -811,10 +797,37 @@ __gen6_read(64) #undef __chv_read #undef __vlv_read #undef __gen6_read -#undef __vgpu_read #undef GEN6_READ_FOOTER #undef GEN6_READ_HEADER +#define VGPU_READ_HEADER(x) \ + unsigned long irqflags; \ + u##x val = 0; \ + assert_device_not_suspended(dev_priv); \ + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) + +#define VGPU_READ_FOOTER \ + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ + trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ + return val + +#define __vgpu_read(x) \ +static u##x \ +vgpu_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \ + VGPU_READ_HEADER(x); \ + val = __raw_i915_read##x(dev_priv, reg); \ + VGPU_READ_FOOTER; \ +} + +__vgpu_read(8) +__vgpu_read(16) +__vgpu_read(32) +__vgpu_read(64) + +#undef __vgpu_read +#undef VGPU_READ_FOOTER +#undef VGPU_READ_HEADER + #define GEN2_WRITE_HEADER \ trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ assert_device_not_suspended(dev_priv); \ @@ -823,7 +836,7 @@ __gen6_read(64) #define __gen2_write(x) \ static void \ -gen2_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ +gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ GEN2_WRITE_HEADER; \ __raw_i915_write##x(dev_priv, reg, val); \ GEN2_WRITE_FOOTER; \ @@ -831,7 +844,7 @@ gen2_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace #define __gen5_write(x) \ static void \ -gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ +gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ GEN2_WRITE_HEADER; \ ilk_dummy_write(dev_priv); \ __raw_i915_write##x(dev_priv, reg, val); \ @@ -854,6 +867,7 @@ __gen2_write(64) #undef GEN2_WRITE_HEADER #define GEN6_WRITE_HEADER \ + u32 offset = i915_mmio_reg_offset(reg); \ unsigned long irqflags; \ trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ assert_device_not_suspended(dev_priv); \ @@ -864,10 +878,10 @@ __gen2_write(64) #define __gen6_write(x) \ static void \ -gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ +gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ u32 __fifo_ret = 0; \ GEN6_WRITE_HEADER; \ - if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ + if (NEEDS_FORCE_WAKE(offset)) { \ __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ } \ __raw_i915_write##x(dev_priv, reg, val); \ @@ -879,10 +893,10 @@ gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace #define __hsw_write(x) \ static void \ -hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ +hsw_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ u32 __fifo_ret = 0; \ GEN6_WRITE_HEADER; \ - if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ + if (NEEDS_FORCE_WAKE(offset)) { \ __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ } \ hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \ @@ -895,15 +909,7 @@ hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) GEN6_WRITE_FOOTER; \ } -#define __vgpu_write(x) \ -static void vgpu_write##x(struct drm_i915_private *dev_priv, \ - off_t reg, u##x val, bool trace) { \ - GEN6_WRITE_HEADER; \ - __raw_i915_write##x(dev_priv, reg, val); \ - GEN6_WRITE_FOOTER; \ -} - -static const u32 gen8_shadowed_regs[] = { +static const i915_reg_t gen8_shadowed_regs[] = { FORCEWAKE_MT, GEN6_RPNSWREQ, GEN6_RC_VIDEO_FREQ, @@ -914,11 +920,12 @@ static const u32 gen8_shadowed_regs[] = { /* TODO: Other registers are not yet used */ }; -static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg) +static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, + i915_reg_t reg) { int i; for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++) - if (reg == gen8_shadowed_regs[i]) + if (i915_mmio_reg_equal(reg, gen8_shadowed_regs[i])) return true; return false; @@ -926,10 +933,10 @@ static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg) #define __gen8_write(x) \ static void \ -gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ +gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ GEN6_WRITE_HEADER; \ hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \ - if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \ + if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(dev_priv, reg)) \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ __raw_i915_write##x(dev_priv, reg, val); \ hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \ @@ -939,22 +946,25 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace #define __chv_write(x) \ static void \ -chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ - bool shadowed = is_gen8_shadowed(dev_priv, reg); \ +chv_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \ + enum forcewake_domains fw_engine = 0; \ GEN6_WRITE_HEADER; \ - if (!shadowed) { \ - if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_RENDER); \ - else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \ - else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \ - __force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \ - } \ + if (!NEEDS_FORCE_WAKE(offset) || \ + is_gen8_shadowed(dev_priv, reg)) \ + fw_engine = 0; \ + else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_RENDER; \ + else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_MEDIA; \ + else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \ + fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \ + if (fw_engine) \ + __force_wake_get(dev_priv, fw_engine); \ __raw_i915_write##x(dev_priv, reg, val); \ GEN6_WRITE_FOOTER; \ } -static const u32 gen9_shadowed_regs[] = { +static const i915_reg_t gen9_shadowed_regs[] = { RING_TAIL(RENDER_RING_BASE), RING_TAIL(GEN6_BSD_RING_BASE), RING_TAIL(VEBOX_RING_BASE), @@ -967,11 +977,12 @@ static const u32 gen9_shadowed_regs[] = { /* TODO: Other registers are not yet used */ }; -static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg) +static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, + i915_reg_t reg) { int i; for (i = 0; i < ARRAY_SIZE(gen9_shadowed_regs); i++) - if (reg == gen9_shadowed_regs[i]) + if (i915_mmio_reg_equal(reg, gen9_shadowed_regs[i])) return true; return false; @@ -979,24 +990,27 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg) #define __gen9_write(x) \ static void \ -gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \ +gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \ bool trace) { \ enum forcewake_domains fw_engine; \ GEN6_WRITE_HEADER; \ - if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \ + hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \ + if (!SKL_NEEDS_FORCE_WAKE(offset) || \ is_gen9_shadowed(dev_priv, reg)) \ fw_engine = 0; \ - else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \ + else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \ fw_engine = FORCEWAKE_RENDER; \ - else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \ + else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \ fw_engine = FORCEWAKE_MEDIA; \ - else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \ + else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \ fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \ else \ fw_engine = FORCEWAKE_BLITTER; \ if (fw_engine) \ __force_wake_get(dev_priv, fw_engine); \ __raw_i915_write##x(dev_priv, reg, val); \ + hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \ + hsw_unclaimed_reg_detect(dev_priv); \ GEN6_WRITE_FOOTER; \ } @@ -1020,20 +1034,41 @@ __gen6_write(8) __gen6_write(16) __gen6_write(32) __gen6_write(64) -__vgpu_write(8) -__vgpu_write(16) -__vgpu_write(32) -__vgpu_write(64) #undef __gen9_write #undef __chv_write #undef __gen8_write #undef __hsw_write #undef __gen6_write -#undef __vgpu_write #undef GEN6_WRITE_FOOTER #undef GEN6_WRITE_HEADER +#define VGPU_WRITE_HEADER \ + unsigned long irqflags; \ + trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ + assert_device_not_suspended(dev_priv); \ + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) + +#define VGPU_WRITE_FOOTER \ + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) + +#define __vgpu_write(x) \ +static void vgpu_write##x(struct drm_i915_private *dev_priv, \ + i915_reg_t reg, u##x val, bool trace) { \ + VGPU_WRITE_HEADER; \ + __raw_i915_write##x(dev_priv, reg, val); \ + VGPU_WRITE_FOOTER; \ +} + +__vgpu_write(8) +__vgpu_write(16) +__vgpu_write(32) +__vgpu_write(64) + +#undef __vgpu_write +#undef VGPU_WRITE_FOOTER +#undef VGPU_WRITE_HEADER + #define ASSIGN_WRITE_MMIO_VFUNCS(x) \ do { \ dev_priv->uncore.funcs.mmio_writeb = x##_write8; \ @@ -1053,7 +1088,8 @@ do { \ static void fw_domain_init(struct drm_i915_private *dev_priv, enum forcewake_domain_id domain_id, - u32 reg_set, u32 reg_ack) + i915_reg_t reg_set, + i915_reg_t reg_ack) { struct intel_uncore_forcewake_domain *d; @@ -1083,8 +1119,6 @@ static void fw_domain_init(struct drm_i915_private *dev_priv, d->reg_post = FORCEWAKE_ACK_VLV; else if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv) || IS_GEN8(dev_priv)) d->reg_post = ECOBUS; - else - d->reg_post = 0; d->i915 = dev_priv; d->id = domain_id; @@ -1198,8 +1232,6 @@ void intel_uncore_init(struct drm_device *dev) switch (INTEL_INFO(dev)->gen) { default: - MISSING_CASE(INTEL_INFO(dev)->gen); - return; case 9: ASSIGN_WRITE_MMIO_VFUNCS(gen9); ASSIGN_READ_MMIO_VFUNCS(gen9); @@ -1260,12 +1292,14 @@ void intel_uncore_fini(struct drm_device *dev) #define GEN_RANGE(l, h) GENMASK(h, l) static const struct register_whitelist { - uint64_t offset; + i915_reg_t offset_ldw, offset_udw; uint32_t size; /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */ uint32_t gen_bitmask; } whitelist[] = { - { RING_TIMESTAMP(RENDER_RING_BASE), 8, GEN_RANGE(4, 9) }, + { .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE), + .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE), + .size = 8, .gen_bitmask = GEN_RANGE(4, 9) }, }; int i915_reg_read_ioctl(struct drm_device *dev, @@ -1275,11 +1309,11 @@ int i915_reg_read_ioctl(struct drm_device *dev, struct drm_i915_reg_read *reg = data; struct register_whitelist const *entry = whitelist; unsigned size; - u64 offset; + i915_reg_t offset_ldw, offset_udw; int i, ret = 0; for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { - if (entry->offset == (reg->offset & -entry->size) && + if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) && (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask)) break; } @@ -1291,27 +1325,28 @@ int i915_reg_read_ioctl(struct drm_device *dev, * be naturally aligned (and those that are not so aligned merely * limit the available flags for that register). */ - offset = entry->offset; + offset_ldw = entry->offset_ldw; + offset_udw = entry->offset_udw; size = entry->size; - size |= reg->offset ^ offset; + size |= reg->offset ^ i915_mmio_reg_offset(offset_ldw); intel_runtime_pm_get(dev_priv); switch (size) { case 8 | 1: - reg->val = I915_READ64_2x32(offset, offset+4); + reg->val = I915_READ64_2x32(offset_ldw, offset_udw); break; case 8: - reg->val = I915_READ64(offset); + reg->val = I915_READ64(offset_ldw); break; case 4: - reg->val = I915_READ(offset); + reg->val = I915_READ(offset_ldw); break; case 2: - reg->val = I915_READ16(offset); + reg->val = I915_READ16(offset_ldw); break; case 1: - reg->val = I915_READ8(offset); + reg->val = I915_READ8(offset_ldw); break; default: ret = -EINVAL; @@ -1427,21 +1462,21 @@ static int ironlake_do_reset(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int ret; - I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, + I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); - ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & + ret = wait_for((I915_READ(ILK_GDSR) & ILK_GRDOM_RESET_ENABLE) == 0, 500); if (ret) return ret; - I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, + I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); - ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & + ret = wait_for((I915_READ(ILK_GDSR) & ILK_GRDOM_RESET_ENABLE) == 0, 500); if (ret) return ret; - I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 0); + I915_WRITE(ILK_GDSR, 0); return 0; } @@ -1467,20 +1502,89 @@ static int gen6_do_reset(struct drm_device *dev) return ret; } -int intel_gpu_reset(struct drm_device *dev) +static int wait_for_register(struct drm_i915_private *dev_priv, + i915_reg_t reg, + const u32 mask, + const u32 value, + const unsigned long timeout_ms) +{ + return wait_for((I915_READ(reg) & mask) == value, timeout_ms); +} + +static int gen8_do_reset(struct drm_device *dev) { - if (INTEL_INFO(dev)->gen >= 6) - return gen6_do_reset(dev); + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *engine; + int i; + + for_each_ring(engine, dev_priv, i) { + I915_WRITE(RING_RESET_CTL(engine->mmio_base), + _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); + + if (wait_for_register(dev_priv, + RING_RESET_CTL(engine->mmio_base), + RESET_CTL_READY_TO_RESET, + RESET_CTL_READY_TO_RESET, + 700)) { + DRM_ERROR("%s: reset request timeout\n", engine->name); + goto not_ready; + } + } + + return gen6_do_reset(dev); + +not_ready: + for_each_ring(engine, dev_priv, i) + I915_WRITE(RING_RESET_CTL(engine->mmio_base), + _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); + + return -EIO; +} + +static int (*intel_get_gpu_reset(struct drm_device *dev))(struct drm_device *) +{ + if (!i915.reset) + return NULL; + + if (INTEL_INFO(dev)->gen >= 8) + return gen8_do_reset; + else if (INTEL_INFO(dev)->gen >= 6) + return gen6_do_reset; else if (IS_GEN5(dev)) - return ironlake_do_reset(dev); + return ironlake_do_reset; else if (IS_G4X(dev)) - return g4x_do_reset(dev); + return g4x_do_reset; else if (IS_G33(dev)) - return g33_do_reset(dev); + return g33_do_reset; else if (INTEL_INFO(dev)->gen >= 3) - return i915_do_reset(dev); + return i915_do_reset; else + return NULL; +} + +int intel_gpu_reset(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + int (*reset)(struct drm_device *); + int ret; + + reset = intel_get_gpu_reset(dev); + if (reset == NULL) return -ENODEV; + + /* If the power well sleeps during the reset, the reset + * request may be dropped and never completes (causing -EIO). + */ + intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); + ret = reset(dev); + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + + return ret; +} + +bool intel_has_gpu_reset(struct drm_device *dev) +{ + return intel_get_gpu_reset(dev) != NULL; } void intel_uncore_check_errors(struct drm_device *dev) |
