diff options
| -rw-r--r-- | drivers/gpu/drm/radeon/evergreen.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/ni.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r100.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r300.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r420.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r520.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/r600.c | 10 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon.h | 28 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_benchmark.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_device.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_fence.c | 232 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_pm.c | 28 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_ring.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_test.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_ttm.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rs400.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rs600.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rs690.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rv515.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/rv770.c | 2 |
20 files changed, 201 insertions, 150 deletions
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c index 5e00d1670aa9..1c7af5654336 100644 --- a/drivers/gpu/drm/radeon/evergreen.c +++ b/drivers/gpu/drm/radeon/evergreen.c @@ -3018,11 +3018,11 @@ restart_ih: case 177: /* CP_INT in IB1 */ case 178: /* CP_INT in IB2 */ DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); - radeon_fence_process(rdev); + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); break; case 181: /* CP EOP event */ DRM_DEBUG("IH: CP EOP\n"); - radeon_fence_process(rdev); + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); break; case 233: /* GUI IDLE */ DRM_DEBUG("IH: GUI idle\n"); @@ -3221,7 +3221,7 @@ int evergreen_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; /* initialize AGP */ diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index 0e5799857465..710ad8c82767 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c @@ -1484,7 +1484,7 @@ int cayman_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 3); if (r) return r; /* initialize memory controller */ diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c index bfc08f6320f8..d2dced5679b5 100644 --- a/drivers/gpu/drm/radeon/r100.c +++ b/drivers/gpu/drm/radeon/r100.c @@ -739,7 +739,7 @@ int r100_irq_process(struct radeon_device *rdev) while (status) { /* SW interrupt */ if (status & RADEON_SW_INT_TEST) { - radeon_fence_process(rdev); + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); } /* gui idle interrupt */ if (status & RADEON_GUI_IDLE_STAT) { @@ -826,7 +826,7 @@ void r100_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0)); radeon_ring_write(rdev, rdev->config.r100.hdp_cntl); /* Emit fence sequence & fire IRQ */ - radeon_ring_write(rdev, PACKET0(rdev->fence_drv.scratch_reg, 0)); + radeon_ring_write(rdev, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0)); radeon_ring_write(rdev, fence->seq); radeon_ring_write(rdev, PACKET0(RADEON_GEN_INT_STATUS, 0)); radeon_ring_write(rdev, RADEON_SW_INT_FIRE); @@ -4048,7 +4048,7 @@ int r100_init(struct radeon_device *rdev) /* initialize VRAM */ r100_mc_init(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c index c93bc64707e1..b04731206460 100644 --- a/drivers/gpu/drm/radeon/r300.c +++ b/drivers/gpu/drm/radeon/r300.c @@ -198,7 +198,7 @@ void r300_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0)); radeon_ring_write(rdev, rdev->config.r300.hdp_cntl); /* Emit fence sequence & fire IRQ */ - radeon_ring_write(rdev, PACKET0(rdev->fence_drv.scratch_reg, 0)); + radeon_ring_write(rdev, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0)); radeon_ring_write(rdev, fence->seq); radeon_ring_write(rdev, PACKET0(RADEON_GEN_INT_STATUS, 0)); radeon_ring_write(rdev, RADEON_SW_INT_FIRE); @@ -1518,7 +1518,7 @@ int r300_init(struct radeon_device *rdev) /* initialize memory controller */ r300_mc_init(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c index 417fab81812f..5dbc378d3c2e 100644 --- a/drivers/gpu/drm/radeon/r420.c +++ b/drivers/gpu/drm/radeon/r420.c @@ -387,7 +387,7 @@ int r420_init(struct radeon_device *rdev) r300_mc_init(rdev); r420_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) { return r; } diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c index 3081d07f8de5..cb96a51f7f7c 100644 --- a/drivers/gpu/drm/radeon/r520.c +++ b/drivers/gpu/drm/radeon/r520.c @@ -278,7 +278,7 @@ int r520_init(struct radeon_device *rdev) r520_mc_init(rdev); rv515_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index 9cdda0b3b081..2fff8cec723c 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -2316,7 +2316,7 @@ void r600_fence_ring_emit(struct radeon_device *rdev, { if (rdev->wb.use_event) { u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET + - (u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base); + (u64)(rdev->fence_drv[fence->ring].scratch_reg - rdev->scratch.reg_base); /* flush read cache over gart */ radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_SYNC, 3)); radeon_ring_write(rdev, PACKET3_TC_ACTION_ENA | @@ -2349,7 +2349,7 @@ void r600_fence_ring_emit(struct radeon_device *rdev, radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit); /* Emit fence sequence & fire IRQ */ radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); - radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2)); + radeon_ring_write(rdev, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2)); radeon_ring_write(rdev, fence->seq); /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */ radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0)); @@ -2575,7 +2575,7 @@ int r600_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; if (rdev->flags & RADEON_IS_AGP) { @@ -3459,11 +3459,11 @@ restart_ih: case 177: /* CP_INT in IB1 */ case 178: /* CP_INT in IB2 */ DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); - radeon_fence_process(rdev); + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); break; case 181: /* CP EOP event */ DRM_DEBUG("IH: CP EOP\n"); - radeon_fence_process(rdev); + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); break; case 233: /* GUI IDLE */ DRM_DEBUG("IH: GUI idle\n"); diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 33f3be369a27..8b93dec66ec6 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -197,7 +197,6 @@ struct radeon_fence_driver { unsigned long last_jiffies; unsigned long last_timeout; wait_queue_head_t queue; - rwlock_t lock; struct list_head created; struct list_head emitted; struct list_head signaled; @@ -212,17 +211,19 @@ struct radeon_fence { uint32_t seq; bool emitted; bool signaled; + /* RB, DMA, etc. */ + int ring; }; -int radeon_fence_driver_init(struct radeon_device *rdev); +int radeon_fence_driver_init(struct radeon_device *rdev, int num_rings); void radeon_fence_driver_fini(struct radeon_device *rdev); -int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence); +int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence, int ring); int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence); -void radeon_fence_process(struct radeon_device *rdev); +void radeon_fence_process(struct radeon_device *rdev, int ring); bool radeon_fence_signaled(struct radeon_fence *fence); int radeon_fence_wait(struct radeon_fence *fence, bool interruptible); -int radeon_fence_wait_next(struct radeon_device *rdev); -int radeon_fence_wait_last(struct radeon_device *rdev); +int radeon_fence_wait_next(struct radeon_device *rdev, int ring); +int radeon_fence_wait_last(struct radeon_device *rdev, int ring); struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence); void radeon_fence_unref(struct radeon_fence **fence); @@ -459,6 +460,18 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc); /* * CP & ring. */ + +/* max number of rings */ +#define RADEON_NUM_RINGS 3 + +/* internal ring indices */ +/* r1xx+ has gfx CP ring */ +#define RADEON_RING_TYPE_GFX_INDEX 0 + +/* cayman has 2 compute CP rings */ +#define CAYMAN_RING_TYPE_CP1_INDEX 1 +#define CAYMAN_RING_TYPE_CP2_INDEX 2 + struct radeon_ib { struct list_head list; unsigned idx; @@ -1235,7 +1248,8 @@ struct radeon_device { struct radeon_mode_info mode_info; struct radeon_scratch scratch; struct radeon_mman mman; - struct radeon_fence_driver fence_drv; + rwlock_t fence_lock; + struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS]; struct radeon_cp cp; /* cayman compute rings */ struct radeon_cp cp1; diff --git a/drivers/gpu/drm/radeon/radeon_benchmark.c b/drivers/gpu/drm/radeon/radeon_benchmark.c index 17e1a9b2d8fb..4f749a69c9c5 100644 --- a/drivers/gpu/drm/radeon/radeon_benchmark.c +++ b/drivers/gpu/drm/radeon/radeon_benchmark.c @@ -43,7 +43,7 @@ static int radeon_benchmark_do_move(struct radeon_device *rdev, unsigned size, start_jiffies = jiffies; for (i = 0; i < n; i++) { - r = radeon_fence_create(rdev, &fence); + r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); if (r) return r; diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 4d66d6868571..4ed4eeb62cdf 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -725,7 +725,7 @@ int radeon_device_init(struct radeon_device *rdev, mutex_init(&rdev->gem.mutex); mutex_init(&rdev->pm.mutex); mutex_init(&rdev->vram_mutex); - rwlock_init(&rdev->fence_drv.lock); + rwlock_init(&rdev->fence_lock); INIT_LIST_HEAD(&rdev->gem.objects); init_waitqueue_head(&rdev->irq.vblank_queue); init_waitqueue_head(&rdev->irq.idle_queue); @@ -857,7 +857,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) struct radeon_device *rdev; struct drm_crtc *crtc; struct drm_connector *connector; - int r; + int i, r; if (dev == NULL || dev->dev_private == NULL) { return -ENODEV; @@ -896,7 +896,8 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state) /* evict vram memory */ radeon_bo_evict_vram(rdev); /* wait for gpu to finish processing current batch */ - radeon_fence_wait_last(rdev); + for (i = 0; i < RADEON_NUM_RINGS; i++) + radeon_fence_wait_last(rdev, i); radeon_save_bios_scratch_regs(rdev); diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 7027766ec2a5..086b8a399118 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -40,32 +40,37 @@ #include "radeon.h" #include "radeon_trace.h" -static void radeon_fence_write(struct radeon_device *rdev, u32 seq) +static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) { + u32 scratch_index; + if (rdev->wb.enabled) { - u32 scratch_index; if (rdev->wb.use_event) - scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; + scratch_index = R600_WB_EVENT_OFFSET + + rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; else - scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; + scratch_index = RADEON_WB_SCRATCH_OFFSET + + rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; rdev->wb.wb[scratch_index/4] = cpu_to_le32(seq); } else - WREG32(rdev->fence_drv.scratch_reg, seq); + WREG32(rdev->fence_drv[ring].scratch_reg, seq); } -static u32 radeon_fence_read(struct radeon_device *rdev) +static u32 radeon_fence_read(struct radeon_device *rdev, int ring) { - u32 seq; + u32 seq = 0; + u32 scratch_index; if (rdev->wb.enabled) { - u32 scratch_index; if (rdev->wb.use_event) - scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; + scratch_index = R600_WB_EVENT_OFFSET + + rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; else - scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; + scratch_index = RADEON_WB_SCRATCH_OFFSET + + rdev->fence_drv[ring].scratch_reg - rdev->scratch.reg_base; seq = le32_to_cpu(rdev->wb.wb[scratch_index/4]); } else - seq = RREG32(rdev->fence_drv.scratch_reg); + seq = RREG32(rdev->fence_drv[ring].scratch_reg); return seq; } @@ -73,28 +78,28 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) { unsigned long irq_flags; - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); if (fence->emitted) { - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } - fence->seq = atomic_add_return(1, &rdev->fence_drv.seq); + fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq); if (!rdev->cp.ready) /* FIXME: cp is not running assume everythings is done right * away */ - radeon_fence_write(rdev, fence->seq); + radeon_fence_write(rdev, fence->seq, fence->ring); else radeon_fence_ring_emit(rdev, fence); trace_radeon_fence_emit(rdev->ddev, fence->seq); fence->emitted = true; - list_move_tail(&fence->list, &rdev->fence_drv.emitted); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } -static bool radeon_fence_poll_locked(struct radeon_device *rdev) +static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) { struct radeon_fence *fence; struct list_head *i, *n; @@ -102,34 +107,34 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev) bool wake = false; unsigned long cjiffies; - seq = radeon_fence_read(rdev); - if (seq != rdev->fence_drv.last_seq) { - rdev->fence_drv.last_seq = seq; - rdev->fence_drv.last_jiffies = jiffies; - rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; + seq = radeon_fence_read(rdev, ring); + if (seq != rdev->fence_drv[ring].last_seq) { + rdev->fence_drv[ring].last_seq = seq; + rdev->fence_drv[ring].last_jiffies = jiffies; + rdev->fence_drv[ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; } else { cjiffies = jiffies; - if (time_after(cjiffies, rdev->fence_drv.last_jiffies)) { - cjiffies -= rdev->fence_drv.last_jiffies; - if (time_after(rdev->fence_drv.last_timeout, cjiffies)) { + if (time_after(cjiffies, rdev->fence_drv[ring].last_jiffies)) { + cjiffies -= rdev->fence_drv[ring].last_jiffies; + if (time_after(rdev->fence_drv[ring].last_timeout, cjiffies)) { /* update the timeout */ - rdev->fence_drv.last_timeout -= cjiffies; + rdev->fence_drv[ring].last_timeout -= cjiffies; } else { /* the 500ms timeout is elapsed we should test * for GPU lockup */ - rdev->fence_drv.last_timeout = 1; + rdev->fence_drv[ring].last_timeout = 1; } } else { /* wrap around update last jiffies, we will just wait * a little longer */ - rdev->fence_drv.last_jiffies = cjiffies; + rdev->fence_drv[ring].last_jiffies = cjiffies; } return false; } n = NULL; - list_for_each(i, &rdev->fence_drv.emitted) { + list_for_each(i, &rdev->fence_drv[ring].emitted) { fence = list_entry(i, struct radeon_fence, list); if (fence->seq == seq) { n = i; @@ -141,11 +146,11 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev) i = n; do { n = i->prev; - list_move_tail(i, &rdev->fence_drv.signaled); + list_move_tail(i, &rdev->fence_drv[ring].signaled); fence = list_entry(i, struct radeon_fence, list); fence->signaled = true; i = n; - } while (i != &rdev->fence_drv.emitted); + } while (i != &rdev->fence_drv[ring].emitted); wake = true; } return wake; @@ -157,14 +162,16 @@ static void radeon_fence_destroy(struct kref *kref) struct radeon_fence *fence; fence = container_of(kref, struct radeon_fence, kref); - write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&fence->rdev->fence_lock, irq_flags); list_del(&fence->list); fence->emitted = false; - write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); + write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags); kfree(fence); } -int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) +int radeon_fence_create(struct radeon_device *rdev, + struct radeon_fence **fence, + int ring) { unsigned long irq_flags; @@ -177,15 +184,15 @@ int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) (*fence)->emitted = false; (*fence)->signaled = false; (*fence)->seq = 0; + (*fence)->ring = ring; INIT_LIST_HEAD(&(*fence)->list); - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - list_add_tail(&(*fence)->list, &rdev->fence_drv.created); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + list_add_tail(&(*fence)->list, &rdev->fence_drv[ring].created); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } - bool radeon_fence_signaled(struct radeon_fence *fence) { unsigned long irq_flags; @@ -197,7 +204,7 @@ bool radeon_fence_signaled(struct radeon_fence *fence) if (fence->rdev->gpu_lockup) return true; - write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&fence->rdev->fence_lock, irq_flags); signaled = fence->signaled; /* if we are shuting down report all fence as signaled */ if (fence->rdev->shutdown) { @@ -208,10 +215,10 @@ bool radeon_fence_signaled(struct radeon_fence *fence) signaled = true; } if (!signaled) { - radeon_fence_poll_locked(fence->rdev); + radeon_fence_poll_locked(fence->rdev, fence->ring); signaled = fence->signaled; } - write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); + write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags); return signaled; } @@ -230,14 +237,14 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) if (radeon_fence_signaled(fence)) { return 0; } - timeout = rdev->fence_drv.last_timeout; + timeout = rdev->fence_drv[fence->ring].last_timeout; retry: /* save current sequence used to check for GPU lockup */ - seq = rdev->fence_drv.last_seq; + seq = rdev->fence_drv[fence->ring].last_seq; trace_radeon_fence_wait_begin(rdev->ddev, seq); if (intr) { radeon_irq_kms_sw_irq_get(rdev); - r = wait_event_interruptible_timeout(rdev->fence_drv.queue, + r = wait_event_interruptible_timeout(rdev->fence_drv[fence->ring].queue, radeon_fence_signaled(fence), timeout); radeon_irq_kms_sw_irq_put(rdev); if (unlikely(r < 0)) { @@ -245,7 +252,7 @@ retry: } } else { radeon_irq_kms_sw_irq_get(rdev); - r = wait_event_timeout(rdev->fence_drv.queue, + r = wait_event_timeout(rdev->fence_drv[fence->ring].queue, radeon_fence_signaled(fence), timeout); radeon_irq_kms_sw_irq_put(rdev); } @@ -258,10 +265,11 @@ retry: timeout = r; goto retry; } - /* don't protect read access to rdev->fence_drv.last_seq + /* don't protect read access to rdev->fence_drv[t].last_seq * if we experiencing a lockup the value doesn't change */ - if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) { + if (seq == rdev->fence_drv[fence->ring].last_seq && + radeon_gpu_is_lockup(rdev)) { /* good news we believe it's a lockup */ printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", fence->seq, seq); @@ -272,20 +280,20 @@ retry: r = radeon_gpu_reset(rdev); if (r) return r; - radeon_fence_write(rdev, fence->seq); + radeon_fence_write(rdev, fence->seq, fence->ring); rdev->gpu_lockup = false; } timeout = RADEON_FENCE_JIFFIES_TIMEOUT; - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; - rdev->fence_drv.last_jiffies = jiffies; - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + rdev->fence_drv[fence->ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; + rdev->fence_drv[fence->ring].last_jiffies = jiffies; + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); goto retry; } return 0; } -int radeon_fence_wait_next(struct radeon_device *rdev) +int radeon_fence_wait_next(struct radeon_device *rdev, int ring) { unsigned long irq_flags; struct radeon_fence *fence; @@ -294,21 +302,21 @@ int radeon_fence_wait_next(struct radeon_device *rdev) if (rdev->gpu_lockup) { return 0; } - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - if (list_empty(&rdev->fence_drv.emitted)) { - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + if (list_empty(&rdev->fence_drv[ring].emitted)) { + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } - fence = list_entry(rdev->fence_drv.emitted.next, + fence = list_entry(rdev->fence_drv[ring].emitted.next, struct radeon_fence, list); radeon_fence_ref(fence); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); r = radeon_fence_wait(fence, false); radeon_fence_unref(&fence); return r; } -int radeon_fence_wait_last(struct radeon_device *rdev) +int radeon_fence_wait_last(struct radeon_device *rdev, int ring) { unsigned long irq_flags; struct radeon_fence *fence; @@ -317,15 +325,15 @@ int radeon_fence_wait_last(struct radeon_device *rdev) if (rdev->gpu_lockup) { return 0; } - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - if (list_empty(&rdev->fence_drv.emitted)) { - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + if (list_empty(&rdev->fence_drv[ring].emitted)) { + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); return 0; } - fence = list_entry(rdev->fence_drv.emitted.prev, + fence = list_entry(rdev->fence_drv[ring].emitted.prev, struct radeon_fence, list); radeon_fence_ref(fence); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); r = radeon_fence_wait(fence, false); radeon_fence_unref(&fence); return r; @@ -347,39 +355,49 @@ void radeon_fence_unref(struct radeon_fence **fence) } } -void radeon_fence_process(struct radeon_device *rdev) +void radeon_fence_process(struct radeon_device *rdev, int ring) { unsigned long irq_flags; bool wake; - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - wake = radeon_fence_poll_locked(rdev); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + wake = radeon_fence_poll_locked(rdev, ring); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); if (wake) { - wake_up_all(&rdev->fence_drv.queue); + wake_up_all(&rdev->fence_drv[ring].queue); } } -int radeon_fence_driver_init(struct radeon_device *rdev) +int radeon_fence_driver_init(struct radeon_device *rdev, int num_rings) { unsigned long irq_flags; - int r; + int r, ring; - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg); - if (r) { - dev_err(rdev->dev, "fence failed to get scratch register\n"); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); - return r; + for (ring = 0; ring < num_rings; ring++) { + write_lock_irqsave(&rdev->fence_lock, irq_flags); + r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); + if (r) { + dev_err(rdev->dev, "fence failed to get scratch register\n"); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); + return r; + } + radeon_fence_write(rdev, 0, ring); + atomic_set(&rdev->fence_drv[ring].seq, 0); + INIT_LIST_HEAD(&rdev->fence_drv[ring].created); + INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); + INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); + init_waitqueue_head(&rdev->fence_drv[ring].queue); + rdev->fence_drv[ring].initialized = true; + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); + } + for (ring = num_rings; ring < RADEON_NUM_RINGS; ring++) { + write_lock_irqsave(&rdev->fence_lock, irq_flags); + INIT_LIST_HEAD(&rdev->fence_drv[ring].created); + INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); + INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); + rdev->fence_drv[ring].initialized = false; + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); } - radeon_fence_write(rdev, 0); - atomic_set(&rdev->fence_drv.seq, 0); - INIT_LIST_HEAD(&rdev->fence_drv.created); - INIT_LIST_HEAD(&rdev->fence_drv.emitted); - INIT_LIST_HEAD(&rdev->fence_drv.signaled); - init_waitqueue_head(&rdev->fence_drv.queue); - rdev->fence_drv.initialized = true; - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); if (radeon_debugfs_fence_init(rdev)) { dev_err(rdev->dev, "fence debugfs file creation failed\n"); } @@ -389,14 +407,17 @@ int radeon_fence_driver_init(struct radeon_device *rdev) void radeon_fence_driver_fini(struct radeon_device *rdev) { unsigned long irq_flags; - - if (!rdev->fence_drv.initialized) - return; - wake_up_all(&rdev->fence_drv.queue); - write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg); - write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); - rdev->fence_drv.initialized = false; + int ring; + + for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { + if (!rdev->fence_drv[ring].initialized) + continue; + wake_up_all(&rdev->fence_drv[ring].queue); + write_lock_irqsave(&rdev->fence_lock, irq_flags); + radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); + write_unlock_irqrestore(&rdev->fence_lock, irq_flags); + rdev->fence_drv[ring].initialized = false; + } } @@ -410,14 +431,21 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) struct drm_device *dev = node->minor->dev; struct radeon_device *rdev = dev->dev_private; struct radeon_fence *fence; - - seq_printf(m, "Last signaled fence 0x%08X\n", - radeon_fence_read(rdev)); - if (!list_empty(&rdev->fence_drv.emitted)) { - fence = list_entry(rdev->fence_drv.emitted.prev, - struct radeon_fence, list); - seq_printf(m, "Last emitted fence %p with 0x%08X\n", - fence, fence->seq); + int i; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + if (!rdev->fence_drv[i].initialized) + continue; + + seq_printf(m, "--- ring %d ---\n", i); + seq_printf(m, "Last signaled fence 0x%08X\n", + radeon_fence_read(rdev, i)); + if (!list_empty(&rdev->fence_drv[i].emitted)) { + fence = list_entry(rdev->fence_drv[i].emitted.prev, + struct radeon_fence, list); + seq_printf(m, "Last emitted fence %p with 0x%08X\n", + fence, fence->seq); + } } return 0; } diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c index 1fb84676afe3..19ed2c6c424a 100644 --- a/drivers/gpu/drm/radeon/radeon_pm.c +++ b/drivers/gpu/drm/radeon/radeon_pm.c @@ -271,7 +271,7 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev) if (rdev->cp.ready) { struct radeon_fence *fence; radeon_ring_alloc(rdev, 64); - radeon_fence_create(rdev, &fence); + radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); radeon_fence_emit(rdev, fence); radeon_ring_commit(rdev); radeon_fence_wait(fence, false); @@ -797,17 +797,25 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work) if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE) { unsigned long irq_flags; int not_processed = 0; - - read_lock_irqsave(&rdev->fence_drv.lock, irq_flags); - if (!list_empty(&rdev->fence_drv.emitted)) { - struct list_head *ptr; - list_for_each(ptr, &rdev->fence_drv.emitted) { - /* count up to 3, that's enought info */ - if (++not_processed >= 3) - break; + int i; + + read_lock_irqsave(&rdev->fence_lock, irq_flags); + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + if (!rdev->fence_drv[i].initialized) + continue; + + if (!list_empty(&rdev->fence_drv[i].emitted)) { + struct list_head *ptr; + list_for_each(ptr, &rdev->fence_drv[i].emitted) { + /* count up to 3, that's enought info */ + if (++not_processed >= 3) + break; + } } + if (not_processed >= 3) + break; } - read_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); + read_unlock_irqrestore(&rdev->fence_lock, irq_flags); if (not_processed >= 3) { /* should upclock */ if (rdev->pm.dynpm_planned_action == DYNPM_ACTION_DOWNCLOCK) { diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index f3d7d224eebf..c232317b1dd2 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -113,7 +113,7 @@ int radeon_ib_get(struct radeon_device *rdev, struct radeon_ib **ib) int r = 0, i, c; *ib = NULL; - r = radeon_fence_create(rdev, &fence); + r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); if (r) { dev_err(rdev->dev, "failed to create fence for new IB\n"); return r; @@ -314,7 +314,7 @@ int radeon_ring_alloc(struct radeon_device *rdev, unsigned ndw) if (ndw < rdev->cp.ring_free_dw) { break; } - r = radeon_fence_wait_next(rdev); + r = radeon_fence_wait_next(rdev, RADEON_RING_TYPE_GFX_INDEX); if (r) return r; } diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c index 602fa3541c45..37f7acb6d5f7 100644 --- a/drivers/gpu/drm/radeon/radeon_test.c +++ b/drivers/gpu/drm/radeon/radeon_test.c @@ -104,7 +104,7 @@ void radeon_test_moves(struct radeon_device *rdev) radeon_bo_kunmap(gtt_obj[i]); - r = radeon_fence_create(rdev, &fence); + r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); if (r) { DRM_ERROR("Failed to create GTT->VRAM fence %d\n", i); goto out_cleanup; @@ -153,7 +153,7 @@ void radeon_test_moves(struct radeon_device *rdev) radeon_bo_kunmap(vram_obj); - r = radeon_fence_create(rdev, &fence); + r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); if (r) { DRM_ERROR("Failed to create VRAM->GTT fence %d\n", i); goto out_cleanup; diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index e111a3812434..112ecaa6362b 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -226,7 +226,7 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, int r; rdev = radeon_get_rdev(bo->bdev); - r = radeon_fence_create(rdev, &fence); + r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX); if (unlikely(r)) { return r; } diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c index 06b90c87f8f3..c71fa16106ca 100644 --- a/drivers/gpu/drm/radeon/rs400.c +++ b/drivers/gpu/drm/radeon/rs400.c @@ -516,7 +516,7 @@ int rs400_init(struct radeon_device *rdev) /* initialize memory controller */ rs400_mc_init(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c index b1053d640423..1c9ab9409531 100644 --- a/drivers/gpu/drm/radeon/rs600.c +++ b/drivers/gpu/drm/radeon/rs600.c @@ -642,7 +642,7 @@ int rs600_irq_process(struct radeon_device *rdev) while (status || rdev->irq.stat_regs.r500.disp_int) { /* SW interrupt */ if (G_000044_SW_INT(status)) { - radeon_fence_process(rdev); + radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); } /* GUI idle */ if (G_000040_GUI_IDLE(status)) { @@ -962,7 +962,7 @@ int rs600_init(struct radeon_device *rdev) rs600_mc_init(rdev); rs600_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c index a9049ed1a519..8aa5e7ef2efc 100644 --- a/drivers/gpu/drm/radeon/rs690.c +++ b/drivers/gpu/drm/radeon/rs690.c @@ -735,7 +735,7 @@ int rs690_init(struct radeon_device *rdev) rs690_mc_init(rdev); rv515_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c index 6613ee9ecca3..fd8da02e1ca5 100644 --- a/drivers/gpu/drm/radeon/rv515.c +++ b/drivers/gpu/drm/radeon/rv515.c @@ -510,7 +510,7 @@ int rv515_init(struct radeon_device *rdev) rv515_mc_init(rdev); rv515_debugfs(rdev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; r = radeon_irq_kms_init(rdev); diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c index 23ae1c60ab3d..8637fd84e499 100644 --- a/drivers/gpu/drm/radeon/rv770.c +++ b/drivers/gpu/drm/radeon/rv770.c @@ -1194,7 +1194,7 @@ int rv770_init(struct radeon_device *rdev) /* Initialize clocks */ radeon_get_clock_info(rdev->ddev); /* Fence driver */ - r = radeon_fence_driver_init(rdev); + r = radeon_fence_driver_init(rdev, 1); if (r) return r; /* initialize AGP */ |
