diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 336 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 23 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 37 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_object.c | 13 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 108 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 12 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 5 |
10 files changed, 321 insertions, 245 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 028ab6007873..3e795a099d06 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -133,12 +133,12 @@ struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_agp_tt_create); -int ttm_agp_tt_populate(struct ttm_tt *ttm) +int ttm_agp_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { if (ttm->state != tt_unpopulated) return 0; - return ttm_pool_populate(ttm); + return ttm_pool_populate(ttm, ctx); } EXPORT_SYMBOL(ttm_agp_tt_populate); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c088703777e2..2eb71ffe95a6 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -42,11 +42,6 @@ #include <linux/atomic.h> #include <linux/reservation.h> -#define TTM_ASSERT_LOCKED(param) -#define TTM_DEBUG(fmt, arg...) -#define TTM_BO_HASH_ORDER 13 - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); static struct attribute ttm_bo_count = { @@ -165,7 +160,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -216,7 +211,7 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); ttm_bo_del_from_lru(bo); ttm_bo_add_to_lru(bo); @@ -233,7 +228,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) int ret = 0; uint32_t page_flags = 0; - TTM_ASSERT_LOCKED(&bo->mutex); + reservation_object_assert_held(bo->resv); bo->ttm = NULL; if (bdev->need_dma32) @@ -269,9 +264,8 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) } static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem, - bool evict, bool interruptible, - bool no_wait_gpu) + struct ttm_mem_reg *mem, bool evict, + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); @@ -306,7 +300,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, goto out_err; if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(bo->ttm, mem); + ret = ttm_tt_bind(bo->ttm, mem, ctx); if (ret) goto out_err; } @@ -325,12 +319,11 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem); + ret = ttm_bo_move_ttm(bo, ctx, mem); else if (bdev->driver->move) - ret = bdev->driver->move(bo, evict, interruptible, - no_wait_gpu, mem); + ret = bdev->driver->move(bo, evict, ctx, mem); else - ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem); + ret = ttm_bo_move_memcpy(bo, ctx, mem); if (ret) { if (bdev->driver->move_notify) { @@ -355,13 +348,13 @@ moved: bo->evicted = false; } - if (bo->mem.mm_node) { + if (bo->mem.mm_node) bo->offset = (bo->mem.start << PAGE_SHIFT) + bdev->man[bo->mem.mem_type].gpu_offset; - bo->cur_placement = bo->mem.placement; - } else + else bo->offset = 0; + ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; return 0; out_err: @@ -390,8 +383,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) ttm_tt_destroy(bo->ttm); bo->ttm = NULL; ttm_bo_mem_put(bo, &bo->mem); - - ww_mutex_unlock (&bo->resv->lock); } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) @@ -448,7 +439,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) } spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); + ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; if (!ret) { if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { ttm_bo_del_from_lru(bo); @@ -457,6 +448,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) reservation_object_unlock(&bo->ttm_resv); ttm_bo_cleanup_memtype_use(bo); + reservation_object_unlock(bo->resv); return; } @@ -472,7 +464,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_add_to_lru(bo); } - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } if (bo->resv != &bo->ttm_resv) reservation_object_unlock(&bo->ttm_resv); @@ -487,20 +479,21 @@ error: } /** - * function ttm_bo_cleanup_refs_and_unlock + * function ttm_bo_cleanup_refs * If bo idle, remove from delayed- and lru lists, and unref. * If not idle, do nothing. * * Must be called with lru_lock and reservation held, this function - * will drop both before returning. + * will drop the lru lock and optionally the reservation lock before returning. * * @interruptible Any sleeps should occur interruptibly. * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + * @unlock_resv Unlock the reservation lock as well. */ -static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_gpu) +static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait_gpu, + bool unlock_resv) { struct ttm_bo_global *glob = bo->glob; struct reservation_object *resv; @@ -518,7 +511,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, if (ret && !no_wait_gpu) { long lret; - ww_mutex_unlock(&bo->resv->lock); + + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); lret = reservation_object_wait_timeout_rcu(resv, true, @@ -531,24 +526,24 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, return -EBUSY; spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); - - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - if (ret) { + if (unlock_resv && !reservation_object_trylock(bo->resv)) { + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ spin_unlock(&glob->lru_lock); return 0; } + ret = 0; } if (ret || unlikely(list_empty(&bo->ddestroy))) { - __ttm_bo_unreserve(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); return ret; } @@ -560,6 +555,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); + return 0; } @@ -567,60 +565,44 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, * Traverse the delayed list, and call ttm_bo_cleanup_refs on all * encountered buffers. */ - -static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) { struct ttm_bo_global *glob = bdev->glob; - struct ttm_buffer_object *entry = NULL; - int ret = 0; - - spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - goto out_unlock; + struct list_head removed; + bool empty; - entry = list_first_entry(&bdev->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&entry->list_kref); + INIT_LIST_HEAD(&removed); - for (;;) { - struct ttm_buffer_object *nentry = NULL; + spin_lock(&glob->lru_lock); + while (!list_empty(&bdev->ddestroy)) { + struct ttm_buffer_object *bo; - if (entry->ddestroy.next != &bdev->ddestroy) { - nentry = list_first_entry(&entry->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&nentry->list_kref); - } + bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, + ddestroy); + kref_get(&bo->list_kref); + list_move_tail(&bo->ddestroy, &removed); - ret = __ttm_bo_reserve(entry, false, true, NULL); - if (remove_all && ret) { + if (remove_all || bo->resv != &bo->ttm_resv) { spin_unlock(&glob->lru_lock); - ret = __ttm_bo_reserve(entry, false, false, NULL); + reservation_object_lock(bo->resv, NULL); + spin_lock(&glob->lru_lock); - } + ttm_bo_cleanup_refs(bo, false, !remove_all, true); - if (!ret) - ret = ttm_bo_cleanup_refs_and_unlock(entry, false, - !remove_all); - else + } else if (reservation_object_trylock(bo->resv)) { + ttm_bo_cleanup_refs(bo, false, !remove_all, true); + } else { spin_unlock(&glob->lru_lock); + } - kref_put(&entry->list_kref, ttm_bo_release_list); - entry = nentry; - - if (ret || !entry) - goto out; - + kref_put(&bo->list_kref, ttm_bo_release_list); spin_lock(&glob->lru_lock); - if (list_empty(&entry->ddestroy)) - break; } - -out_unlock: + list_splice_tail(&removed, &bdev->ddestroy); + empty = list_empty(&bdev->ddestroy); spin_unlock(&glob->lru_lock); -out: - if (entry) - kref_put(&entry->list_kref, ttm_bo_release_list); - return ret; + + return empty; } static void ttm_bo_delayed_workqueue(struct work_struct *work) @@ -628,7 +610,7 @@ static void ttm_bo_delayed_workqueue(struct work_struct *work) struct ttm_bo_device *bdev = container_of(work, struct ttm_bo_device, wq.work); - if (ttm_bo_delayed_delete(bdev, false)) { + if (!ttm_bo_delayed_delete(bdev, false)) { schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); } @@ -672,15 +654,15 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) } EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); -static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, - bool no_wait_gpu) +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_reg evict_mem; struct ttm_placement placement; int ret = 0; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -690,8 +672,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, placement.num_placement = 0; placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); - ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, - no_wait_gpu); + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { pr_err("Failed to find memory space for buffer 0x%p eviction\n", @@ -701,8 +682,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, - no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); if (unlikely(ret)) { if (ret != -ERESTARTSYS) pr_err("Buffer eviction failed\n"); @@ -728,49 +708,78 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_eviction_valuable); +/** + * Check the target bo is allowable to be evicted or swapout, including cases: + * + * a. if share same reservation object with ctx->resv, have assumption + * reservation objects should already be locked, so not lock again and + * return true directly when either the opreation allow_reserved_eviction + * or the target bo already is in delayed free list; + * + * b. Otherwise, trylock it. + */ +static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, bool *locked) +{ + bool ret = false; + + *locked = false; + if (bo->resv == ctx->resv) { + reservation_object_assert_held(bo->resv); + if (ctx->allow_reserved_eviction || !list_empty(&bo->ddestroy)) + ret = true; + } else { + *locked = reservation_object_trylock(bo->resv); + ret = *locked; + } + + return ret; +} + static int ttm_mem_evict_first(struct ttm_bo_device *bdev, - uint32_t mem_type, - const struct ttm_place *place, - bool interruptible, - bool no_wait_gpu) + uint32_t mem_type, + const struct ttm_place *place, + struct ttm_operation_ctx *ctx) { struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - struct ttm_buffer_object *bo; - int ret = -EBUSY; + struct ttm_buffer_object *bo = NULL; + bool locked = false; unsigned i; + int ret; spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &man->lru[i], lru) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (ret) + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) continue; if (place && !bdev->driver->eviction_valuable(bo, place)) { - __ttm_bo_unreserve(bo); - ret = -EBUSY; + if (locked) + reservation_object_unlock(bo->resv); continue; } - break; } - if (!ret) + /* If the inner loop terminated early, we have our candidate */ + if (&bo->lru != &man->lru[i]) break; + + bo = NULL; } - if (ret) { + if (!bo) { spin_unlock(&glob->lru_lock); - return ret; + return -EBUSY; } kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, - no_wait_gpu); + ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, + ctx->no_wait_gpu, locked); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -778,10 +787,14 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); - BUG_ON(ret != 0); - - ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); - ttm_bo_unreserve(bo); + ret = ttm_bo_evict(bo, ctx); + if (locked) { + ttm_bo_unreserve(bo); + } else { + spin_lock(&glob->lru_lock); + ttm_bo_add_to_lru(bo); + spin_unlock(&glob->lru_lock); + } kref_put(&bo->list_kref, ttm_bo_release_list); return ret; @@ -832,8 +845,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, uint32_t mem_type, const struct ttm_place *place, struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; @@ -845,8 +857,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, mem_type, place, - interruptible, no_wait_gpu); + ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); if (unlikely(ret != 0)) return ret; } while (1); @@ -909,8 +920,7 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; @@ -1004,8 +1014,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, return 0; } - ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, - interruptible, no_wait_gpu); + ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; @@ -1024,14 +1033,13 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, EXPORT_SYMBOL(ttm_bo_mem_space); static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret = 0; struct ttm_mem_reg mem; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; @@ -1041,12 +1049,10 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, /* * Determine where to move the buffer. */ - ret = ttm_bo_mem_space(bo, placement, &mem, - interruptible, no_wait_gpu); + ret = ttm_bo_mem_space(bo, placement, &mem, ctx); if (ret) goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, - interruptible, no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); out_unlock: if (ret && mem.mm_node) ttm_bo_mem_put(bo, &mem); @@ -1097,20 +1103,18 @@ bool ttm_bo_mem_compat(struct ttm_placement *placement, EXPORT_SYMBOL(ttm_bo_mem_compat); int ttm_bo_validate(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret; uint32_t new_flags; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); /* * Check whether we need to move buffer. */ if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { - ret = ttm_bo_move_buffer(bo, placement, interruptible, - no_wait_gpu); + ret = ttm_bo_move_buffer(bo, placement, ctx); if (ret) return ret; } else { @@ -1139,7 +1143,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - bool interruptible, + struct ttm_operation_ctx *ctx, struct file *persistent_swap_storage, size_t acc_size, struct sg_table *sg, @@ -1151,7 +1155,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; bool locked; - ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); + ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); if (ret) { pr_err("Out of kernel memory\n"); if (destroy) @@ -1199,7 +1203,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->sg = sg; if (resv) { bo->resv = resv; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); } else { bo->resv = &bo->ttm_resv; } @@ -1221,12 +1225,12 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, * since otherwise lockdep will be angered in radeon. */ if (!resv) { - locked = ww_mutex_trylock(&bo->resv->lock); + locked = reservation_object_trylock(bo->resv); WARN_ON(!locked); } if (likely(!ret)) - ret = ttm_bo_validate(bo, placement, interruptible, false); + ret = ttm_bo_validate(bo, placement, ctx); if (unlikely(ret)) { if (!resv) @@ -1259,10 +1263,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev, struct reservation_object *resv, void (*destroy) (struct ttm_buffer_object *)) { + struct ttm_operation_ctx ctx = { interruptible, false }; int ret; ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, interruptible, + page_alignment, &ctx, persistent_swap_storage, acc_size, sg, resv, destroy); if (ret) @@ -1334,6 +1339,7 @@ EXPORT_SYMBOL(ttm_bo_create); static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, unsigned mem_type) { + struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_bo_global *glob = bdev->glob; struct dma_fence *fence; @@ -1348,7 +1354,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { while (!list_empty(&man->lru[i])) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); if (ret) return ret; spin_lock(&glob->lru_lock); @@ -1469,7 +1475,6 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj) struct ttm_bo_global *glob = container_of(kobj, struct ttm_bo_global, kobj); - ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); __free_page(glob->dummy_read_page); kfree(glob); } @@ -1494,6 +1499,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) mutex_init(&glob->device_list_mutex); spin_lock_init(&glob->lru_lock); glob->mem_glob = bo_ref->mem_glob; + glob->mem_glob->bo_glob = glob; glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); if (unlikely(glob->dummy_read_page == NULL)) { @@ -1504,14 +1510,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) INIT_LIST_HEAD(&glob->swap_lru[i]); INIT_LIST_HEAD(&glob->device_list); - - ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); - ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); - if (unlikely(ret != 0)) { - pr_err("Could not register buffer object swapout\n"); - goto out_no_shrink; - } - atomic_set(&glob->bo_count, 0); ret = kobject_init_and_add( @@ -1519,8 +1517,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) if (unlikely(ret != 0)) kobject_put(&glob->kobj); return ret; -out_no_shrink: - __free_page(glob->dummy_read_page); out_no_drp: kfree(glob); return ret; @@ -1554,16 +1550,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) cancel_delayed_work_sync(&bdev->wq); - while (ttm_bo_delayed_delete(bdev, true)) - ; + if (ttm_bo_delayed_delete(bdev, true)) + pr_debug("Delayed destroy list was clean\n"); spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - TTM_DEBUG("Delayed destroy list was clean\n"); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) if (list_empty(&bdev->man[0].lru[0])) - TTM_DEBUG("Swap list %d was clean\n", i); + pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); drm_vma_offset_manager_destroy(&bdev->vma_manager); @@ -1706,21 +1699,20 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release); * A buffer object shrink method that tries to swap out the first * buffer object on the bo_global::swap_lru list. */ - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) +int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) { - struct ttm_bo_global *glob = - container_of(shrink, struct ttm_bo_global, shrink); struct ttm_buffer_object *bo; int ret = -EBUSY; + bool locked; unsigned i; spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &glob->swap_lru[i], swap) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (!ret) + if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { + ret = 0; break; + } } if (!ret) break; @@ -1734,7 +1726,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); + ret = ttm_bo_cleanup_refs(bo, false, false, true); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1748,6 +1740,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) if (bo->mem.mem_type != TTM_PL_SYSTEM || bo->ttm->caching_state != tt_cached) { + struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_reg evict_mem; evict_mem = bo->mem; @@ -1755,8 +1748,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; evict_mem.mem_type = TTM_PL_SYSTEM; - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, - false, false); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); if (unlikely(ret != 0)) goto out; } @@ -1788,14 +1780,20 @@ out: * already swapped buffer. */ - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } +EXPORT_SYMBOL(ttm_bo_swapout); void ttm_bo_swapout_all(struct ttm_bo_device *bdev) { - while (ttm_bo_swapout(&bdev->glob->shrink) == 0) + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + + while (ttm_bo_swapout(bdev->glob, &ctx) == 0) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); @@ -1822,10 +1820,12 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) return -ERESTARTSYS; if (!ww_mutex_is_locked(&bo->resv->lock)) goto out_unlock; - ret = __ttm_bo_reserve(bo, true, false, NULL); + ret = reservation_object_lock_interruptible(bo->resv, NULL); + if (ret == -EINTR) + ret = -ERESTARTSYS; if (unlikely(ret != 0)) goto out_unlock; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); out_unlock: mutex_unlock(&bo->wu_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index e7a519f1849b..153de1bf0232 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -45,7 +45,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) } int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; @@ -53,7 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) @@ -73,7 +73,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, return ret; if (new_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(ttm, new_mem); + ret = ttm_tt_bind(ttm, new_mem, ctx); if (unlikely(ret != 0)) return ret; } @@ -329,7 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; @@ -345,7 +345,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, unsigned long add = 0; int dir; - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (ret) return ret; @@ -376,7 +376,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, * TTM might be null for moves within the same region. */ if (ttm && ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); if (ret) goto out1; } @@ -485,7 +485,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->acc_size = 0; fbo->resv = &fbo->ttm_resv; reservation_object_init(fbo->resv); - ret = ww_mutex_trylock(&fbo->resv->lock); + ret = reservation_object_trylock(fbo->resv); WARN_ON(!ret); *new_obj = fbo; @@ -545,14 +545,19 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, unsigned long num_pages, struct ttm_bo_kmap_obj *map) { - struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; + struct ttm_mem_reg *mem = &bo->mem; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; struct ttm_tt *ttm = bo->ttm; + pgprot_t prot; int ret; BUG_ON(!ttm); if (ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, &ctx); if (ret) return ret; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index c8ebb757e36b..8e68e70b02b9 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -92,6 +92,17 @@ out_unlock: return ret; } +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_bo_device *bdev = bo->bdev; + + if (bdev->driver->io_mem_pfn) + return bdev->driver->io_mem_pfn(bo, page_offset); + + return ttm_bo_default_io_mem_pfn(bo, page_offset); +} + static int ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -215,12 +226,17 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); } else { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + ttm = bo->ttm; cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); /* Allocate all page at once, most common usage */ - if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + if (ttm->bdev->driver->ttm_tt_populate(ttm, &ctx)) { retval = VM_FAULT_OOM; goto out_io_unlock; } @@ -234,7 +250,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (bo->mem.bus.is_iomem) { /* Iomem should not be marked encrypted */ cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); - pfn = bdev->driver->io_mem_pfn(bo, page_offset); + pfn = ttm_bo_io_mem_pfn(bo, page_offset); } else { page = ttm->pages[page_offset]; if (unlikely(!page && i == 0)) { diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 5e1bcabffef5..373ced0b2fc2 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -38,7 +38,7 @@ static void ttm_eu_backoff_reservation_reverse(struct list_head *list, list_for_each_entry_continue_reverse(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } } @@ -69,7 +69,7 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, struct ttm_buffer_object *bo = entry->bo; ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); @@ -112,7 +112,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) { - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); ret = -EBUSY; @@ -203,7 +203,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, else reservation_object_add_excl_fence(bo->resv, fence); ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); if (ticket) diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index e96374990398..aa0c38136958 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -211,35 +211,33 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, */ static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra) + uint64_t extra, struct ttm_operation_ctx *ctx) { int ret; - struct ttm_mem_shrink *shrink; spin_lock(&glob->lock); - if (glob->shrink == NULL) - goto out; while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - shrink = glob->shrink; spin_unlock(&glob->lock); - ret = shrink->do_shrink(shrink); + ret = ttm_bo_swapout(glob->bo_glob, ctx); spin_lock(&glob->lock); if (unlikely(ret != 0)) - goto out; + break; } -out: + spin_unlock(&glob->lock); } - - static void ttm_shrink_work(struct work_struct *work) { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; struct ttm_mem_global *glob = container_of(work, struct ttm_mem_global, work); - ttm_shrink(glob, true, 0ULL); + ttm_shrink(glob, true, 0ULL, &ctx); } static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, @@ -514,7 +512,7 @@ out_unlock: static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, struct ttm_mem_zone *single_zone, uint64_t memory, - bool no_wait, bool interruptible) + struct ttm_operation_ctx *ctx) { int count = TTM_MEMORY_ALLOC_RETRIES; @@ -522,33 +520,32 @@ static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, single_zone, memory, true) != 0)) { - if (no_wait) + if (ctx->no_wait_gpu) return -ENOMEM; if (unlikely(count-- == 0)) return -ENOMEM; - ttm_shrink(glob, false, memory + (memory >> 2) + 16); + ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); } return 0; } int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, - bool no_wait, bool interruptible) + struct ttm_operation_ctx *ctx) { /** * Normal allocations of kernel memory are registered in * all zones. */ - return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, - interruptible); + return ttm_mem_global_alloc_zone(glob, NULL, memory, ctx); } EXPORT_SYMBOL(ttm_mem_global_alloc); int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, - struct page *page, uint64_t size) + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx) { - struct ttm_mem_zone *zone = NULL; /** @@ -563,7 +560,7 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) zone = glob->zone_kernel; #endif - return ttm_mem_global_alloc_zone(glob, zone, size, false, false); + return ttm_mem_global_alloc_zone(glob, zone, size, ctx); } void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index 26a7ad0f4789..1aa2baa83959 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -325,6 +325,10 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_ref_object *ref; struct drm_hash_item *hash; struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; int ret = -EINVAL; if (base->tfile != tfile && !base->shareable) @@ -350,7 +354,7 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, return -EPERM; ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), - false, false); + &ctx); if (unlikely(ret != 0)) return ret; ref = kmalloc(sizeof(*ref), GFP_KERNEL); @@ -686,7 +690,10 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, dma_buf = prime->dma_buf; if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) { DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false + }; exp_info.ops = &tdev->ops; exp_info.size = prime->size; exp_info.flags = flags; @@ -696,7 +703,7 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, * Need to create a new dma_buf, with memory accounting. */ ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size, - false, true); + &ctx); if (unlikely(ret != 0)) { mutex_unlock(&prime->mutex); goto out_unref; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 8d7172e8381d..f1a3d55ead83 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -81,6 +81,7 @@ struct ttm_page_pool { char *name; unsigned long nfrees; unsigned long nrefills; + unsigned int order; }; /** @@ -222,6 +223,17 @@ static struct kobj_type ttm_pool_kobj_type = { static struct ttm_pool_manager *_manager; #ifndef CONFIG_X86 +static int set_pages_wb(struct page *page, int numpages) +{ +#if IS_ENABLED(CONFIG_AGP) + int i; + + for (i = 0; i < numpages; i++) + unmap_page_from_agp(page++); +#endif + return 0; +} + static int set_pages_array_wb(struct page **pages, int addrinarray) { #if IS_ENABLED(CONFIG_AGP) @@ -284,13 +296,23 @@ static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, } /* set memory back to wb and free the pages. */ -static void ttm_pages_put(struct page *pages[], unsigned npages) +static void ttm_pages_put(struct page *pages[], unsigned npages, + unsigned int order) { - unsigned i; - if (set_pages_array_wb(pages, npages)) - pr_err("Failed to set %d pages to wb!\n", npages); - for (i = 0; i < npages; ++i) - __free_page(pages[i]); + unsigned int i, pages_nr = (1 << order); + + if (order == 0) { + if (set_pages_array_wb(pages, npages)) + pr_err("Failed to set %d pages to wb!\n", npages); + } + + for (i = 0; i < npages; ++i) { + if (order > 0) { + if (set_pages_wb(pages[i], pages_nr)) + pr_err("Failed to set %d pages to wb!\n", pages_nr); + } + __free_pages(pages[i], order); + } } static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, @@ -353,7 +375,7 @@ restart: */ spin_unlock_irqrestore(&pool->lock, irq_flags); - ttm_pages_put(pages_to_free, freed_pages); + ttm_pages_put(pages_to_free, freed_pages, pool->order); if (likely(nr_free != FREE_ALL_PAGES)) nr_free -= freed_pages; @@ -388,7 +410,7 @@ restart: spin_unlock_irqrestore(&pool->lock, irq_flags); if (freed_pages) - ttm_pages_put(pages_to_free, freed_pages); + ttm_pages_put(pages_to_free, freed_pages, pool->order); out: if (pages_to_free != static_buf) kfree(pages_to_free); @@ -412,6 +434,7 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) struct ttm_page_pool *pool; int shrink_pages = sc->nr_to_scan; unsigned long freed = 0; + unsigned int nr_free_pool; if (!mutex_trylock(&lock)) return SHRINK_STOP; @@ -419,12 +442,20 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) /* select start pool in round robin fashion */ for (i = 0; i < NUM_POOLS; ++i) { unsigned nr_free = shrink_pages; + unsigned page_nr; + if (shrink_pages == 0) break; + pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; + page_nr = (1 << pool->order); /* OK to use static buffer since global mutex is held. */ - shrink_pages = ttm_page_pool_free(pool, nr_free, true); - freed += nr_free - shrink_pages; + nr_free_pool = roundup(nr_free, page_nr) >> pool->order; + shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); + freed += (nr_free_pool - shrink_pages) << pool->order; + if (freed >= sc->nr_to_scan) + break; + shrink_pages <<= pool->order; } mutex_unlock(&lock); return freed; @@ -436,9 +467,12 @@ ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { unsigned i; unsigned long count = 0; + struct ttm_page_pool *pool; - for (i = 0; i < NUM_POOLS; ++i) - count += _manager->pools[i].npages; + for (i = 0; i < NUM_POOLS; ++i) { + pool = &_manager->pools[i]; + count += (pool->npages << pool->order); + } return count; } @@ -510,8 +544,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, int r = 0; unsigned i, j, cpages; unsigned npages = 1 << order; - unsigned max_cpages = min(count, - (unsigned)(PAGE_SIZE/sizeof(struct page *))); + unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); /* allocate array for page caching change */ caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); @@ -845,7 +878,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, #endif struct list_head plist; struct page *p = NULL; - unsigned count; + unsigned count, first; int r; /* No pool for cached pages */ @@ -886,6 +919,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, } #endif + first = i; while (npages) { p = alloc_page(gfp_flags); if (!p) { @@ -893,6 +927,10 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, return -ENOMEM; } + /* Swap the pages if we detect consecutive order */ + if (i > first && pages[i - 1] == p - 1) + swap(p, pages[i - 1]); + pages[i++] = p; --npages; } @@ -921,8 +959,15 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages - count, 0); - list_for_each_entry(p, &plist, lru) - pages[count++] = p; + first = count; + list_for_each_entry(p, &plist, lru) { + struct page *tmp = p; + + /* Swap the pages if we detect consecutive order */ + if (count > first && pages[count - 1] == tmp - 1) + swap(tmp, pages[count - 1]); + pages[count++] = tmp; + } if (r) { /* If there is any pages in the list put them back to @@ -937,7 +982,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, } static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, - char *name) + char *name, unsigned int order) { spin_lock_init(&pool->lock); pool->fill_lock = false; @@ -945,11 +990,17 @@ static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, pool->npages = pool->nfrees = 0; pool->gfp_flags = flags; pool->name = name; + pool->order = order; } int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) { int ret; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + unsigned order = HPAGE_PMD_ORDER; +#else + unsigned order = 0; +#endif WARN_ON(_manager); @@ -957,23 +1008,23 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); - ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); + ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); - ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); + ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); ttm_page_pool_init_locked(&_manager->wc_pool_dma32, - GFP_USER | GFP_DMA32, "wc dma"); + GFP_USER | GFP_DMA32, "wc dma", 0); ttm_page_pool_init_locked(&_manager->uc_pool_dma32, - GFP_USER | GFP_DMA32, "uc dma"); + GFP_USER | GFP_DMA32, "uc dma", 0); ttm_page_pool_init_locked(&_manager->wc_pool_huge, GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP), - "wc huge"); + "wc huge", order); ttm_page_pool_init_locked(&_manager->uc_pool_huge, GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP) - , "uc huge"); + , "uc huge", order); _manager->options.max_size = max_pages; _manager->options.small = SMALL_ALLOCATION; @@ -1007,7 +1058,7 @@ void ttm_page_alloc_fini(void) _manager = NULL; } -int ttm_pool_populate(struct ttm_tt *ttm) +int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; unsigned i; @@ -1025,7 +1076,7 @@ int ttm_pool_populate(struct ttm_tt *ttm) for (i = 0; i < ttm->num_pages; ++i) { ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - PAGE_SIZE); + PAGE_SIZE, ctx); if (unlikely(ret != 0)) { ttm_pool_unpopulate(ttm); return -ENOMEM; @@ -1062,12 +1113,13 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_pool_unpopulate); -int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt) +int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + struct ttm_operation_ctx *ctx) { unsigned i, j; int r; - r = ttm_pool_populate(&tt->ttm); + r = ttm_pool_populate(&tt->ttm, ctx); if (r) return r; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c index 6b2627fe9bc1..3ac53918881e 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -923,7 +923,8 @@ static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) * On success pages list will hold count number of correctly * cached pages. On failure will hold the negative return value (-ENOMEM, etc). */ -int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + struct ttm_operation_ctx *ctx) { struct ttm_tt *ttm = &ttm_dma->ttm; struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; @@ -962,7 +963,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) break; ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size); + pool->size, ctx); if (unlikely(ret != 0)) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; @@ -998,7 +999,7 @@ skip_huge: } ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size); + pool->size, ctx); if (unlikely(ret != 0)) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; @@ -1244,15 +1245,12 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) { struct device_pools *p; struct dma_pool *pool = NULL; - char *h[] = {"pool", "refills", "pages freed", "inuse", "available", - "name", "virt", "busaddr"}; if (!_manager) { seq_printf(m, "No pool allocator running.\n"); return 0; } - seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", - h[0], h[1], h[2], h[3], h[4], h[5]); + seq_printf(m, " pool refills pages freed inuse available name\n"); mutex_lock(&_manager->lock); list_for_each_entry(p, &_manager->pools, pools) { struct device *dev = p->dev; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 8ebc8d3560c3..5a046a3c543a 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -261,7 +261,8 @@ void ttm_tt_unbind(struct ttm_tt *ttm) } } -int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) +int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, + struct ttm_operation_ctx *ctx) { int ret = 0; @@ -271,7 +272,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) if (ttm->state == tt_bound) return 0; - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); if (ret) return ret; |
