diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 78 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 13 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 37 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_object.c | 13 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 9 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 5 |
9 files changed, 115 insertions, 71 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 028ab6007873..3e795a099d06 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -133,12 +133,12 @@ struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_agp_tt_create); -int ttm_agp_tt_populate(struct ttm_tt *ttm) +int ttm_agp_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { if (ttm->state != tt_unpopulated) return 0; - return ttm_pool_populate(ttm); + return ttm_pool_populate(ttm, ctx); } EXPORT_SYMBOL(ttm_agp_tt_populate); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 60bb5c12b568..2eb71ffe95a6 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -42,7 +42,6 @@ #include <linux/atomic.h> #include <linux/reservation.h> -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); static struct attribute ttm_bo_count = { @@ -301,7 +300,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, goto out_err; if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(bo->ttm, mem); + ret = ttm_tt_bind(bo->ttm, mem, ctx); if (ret) goto out_err; } @@ -592,6 +591,8 @@ static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) } else if (reservation_object_trylock(bo->resv)) { ttm_bo_cleanup_refs(bo, false, !remove_all, true); + } else { + spin_unlock(&glob->lru_lock); } kref_put(&bo->list_kref, ttm_bo_release_list); @@ -707,6 +708,34 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_eviction_valuable); +/** + * Check the target bo is allowable to be evicted or swapout, including cases: + * + * a. if share same reservation object with ctx->resv, have assumption + * reservation objects should already be locked, so not lock again and + * return true directly when either the opreation allow_reserved_eviction + * or the target bo already is in delayed free list; + * + * b. Otherwise, trylock it. + */ +static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, bool *locked) +{ + bool ret = false; + + *locked = false; + if (bo->resv == ctx->resv) { + reservation_object_assert_held(bo->resv); + if (ctx->allow_reserved_eviction || !list_empty(&bo->ddestroy)) + ret = true; + } else { + *locked = reservation_object_trylock(bo->resv); + ret = *locked; + } + + return ret; +} + static int ttm_mem_evict_first(struct ttm_bo_device *bdev, uint32_t mem_type, const struct ttm_place *place, @@ -722,21 +751,13 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &man->lru[i], lru) { - if (bo->resv == ctx->resv) { - if (!ctx->allow_reserved_eviction && - list_empty(&bo->ddestroy)) - continue; - } else { - locked = reservation_object_trylock(bo->resv); - if (!locked) - continue; - } + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) + continue; if (place && !bdev->driver->eviction_valuable(bo, place)) { if (locked) reservation_object_unlock(bo->resv); - locked = false; continue; } break; @@ -1134,7 +1155,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; bool locked; - ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); + ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); if (ret) { pr_err("Out of kernel memory\n"); if (destroy) @@ -1454,7 +1475,6 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj) struct ttm_bo_global *glob = container_of(kobj, struct ttm_bo_global, kobj); - ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); __free_page(glob->dummy_read_page); kfree(glob); } @@ -1479,6 +1499,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) mutex_init(&glob->device_list_mutex); spin_lock_init(&glob->lru_lock); glob->mem_glob = bo_ref->mem_glob; + glob->mem_glob->bo_glob = glob; glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); if (unlikely(glob->dummy_read_page == NULL)) { @@ -1489,14 +1510,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) INIT_LIST_HEAD(&glob->swap_lru[i]); INIT_LIST_HEAD(&glob->device_list); - - ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); - ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); - if (unlikely(ret != 0)) { - pr_err("Could not register buffer object swapout\n"); - goto out_no_shrink; - } - atomic_set(&glob->bo_count, 0); ret = kobject_init_and_add( @@ -1504,8 +1517,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) if (unlikely(ret != 0)) kobject_put(&glob->kobj); return ret; -out_no_shrink: - __free_page(glob->dummy_read_page); out_no_drp: kfree(glob); return ret; @@ -1688,21 +1699,20 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release); * A buffer object shrink method that tries to swap out the first * buffer object on the bo_global::swap_lru list. */ - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) +int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) { - struct ttm_bo_global *glob = - container_of(shrink, struct ttm_bo_global, shrink); struct ttm_buffer_object *bo; int ret = -EBUSY; + bool locked; unsigned i; spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &glob->swap_lru[i], swap) { - ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; - if (!ret) + if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { + ret = 0; break; + } } if (!ret) break; @@ -1774,10 +1784,16 @@ out: kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } +EXPORT_SYMBOL(ttm_bo_swapout); void ttm_bo_swapout_all(struct ttm_bo_device *bdev) { - while (ttm_bo_swapout(&bdev->glob->shrink) == 0) + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + + while (ttm_bo_swapout(bdev->glob, &ctx) == 0) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 6e353df4e4bc..153de1bf0232 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -73,7 +73,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, return ret; if (new_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(ttm, new_mem); + ret = ttm_tt_bind(ttm, new_mem, ctx); if (unlikely(ret != 0)) return ret; } @@ -376,7 +376,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, * TTM might be null for moves within the same region. */ if (ttm && ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); if (ret) goto out1; } @@ -545,14 +545,19 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, unsigned long num_pages, struct ttm_bo_kmap_obj *map) { - struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; + struct ttm_mem_reg *mem = &bo->mem; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; struct ttm_tt *ttm = bo->ttm; + pgprot_t prot; int ret; BUG_ON(!ttm); if (ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, &ctx); if (ret) return ret; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index c8ebb757e36b..8e68e70b02b9 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -92,6 +92,17 @@ out_unlock: return ret; } +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_bo_device *bdev = bo->bdev; + + if (bdev->driver->io_mem_pfn) + return bdev->driver->io_mem_pfn(bo, page_offset); + + return ttm_bo_default_io_mem_pfn(bo, page_offset); +} + static int ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -215,12 +226,17 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); } else { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + ttm = bo->ttm; cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); /* Allocate all page at once, most common usage */ - if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + if (ttm->bdev->driver->ttm_tt_populate(ttm, &ctx)) { retval = VM_FAULT_OOM; goto out_io_unlock; } @@ -234,7 +250,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (bo->mem.bus.is_iomem) { /* Iomem should not be marked encrypted */ cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); - pfn = bdev->driver->io_mem_pfn(bo, page_offset); + pfn = ttm_bo_io_mem_pfn(bo, page_offset); } else { page = ttm->pages[page_offset]; if (unlikely(!page && i == 0)) { diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index e96374990398..aa0c38136958 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -211,35 +211,33 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, */ static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra) + uint64_t extra, struct ttm_operation_ctx *ctx) { int ret; - struct ttm_mem_shrink *shrink; spin_lock(&glob->lock); - if (glob->shrink == NULL) - goto out; while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - shrink = glob->shrink; spin_unlock(&glob->lock); - ret = shrink->do_shrink(shrink); + ret = ttm_bo_swapout(glob->bo_glob, ctx); spin_lock(&glob->lock); if (unlikely(ret != 0)) - goto out; + break; } -out: + spin_unlock(&glob->lock); } - - static void ttm_shrink_work(struct work_struct *work) { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; struct ttm_mem_global *glob = container_of(work, struct ttm_mem_global, work); - ttm_shrink(glob, true, 0ULL); + ttm_shrink(glob, true, 0ULL, &ctx); } static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, @@ -514,7 +512,7 @@ out_unlock: static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, struct ttm_mem_zone *single_zone, uint64_t memory, - bool no_wait, bool interruptible) + struct ttm_operation_ctx *ctx) { int count = TTM_MEMORY_ALLOC_RETRIES; @@ -522,33 +520,32 @@ static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, single_zone, memory, true) != 0)) { - if (no_wait) + if (ctx->no_wait_gpu) return -ENOMEM; if (unlikely(count-- == 0)) return -ENOMEM; - ttm_shrink(glob, false, memory + (memory >> 2) + 16); + ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); } return 0; } int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, - bool no_wait, bool interruptible) + struct ttm_operation_ctx *ctx) { /** * Normal allocations of kernel memory are registered in * all zones. */ - return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, - interruptible); + return ttm_mem_global_alloc_zone(glob, NULL, memory, ctx); } EXPORT_SYMBOL(ttm_mem_global_alloc); int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, - struct page *page, uint64_t size) + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx) { - struct ttm_mem_zone *zone = NULL; /** @@ -563,7 +560,7 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) zone = glob->zone_kernel; #endif - return ttm_mem_global_alloc_zone(glob, zone, size, false, false); + return ttm_mem_global_alloc_zone(glob, zone, size, ctx); } void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index 26a7ad0f4789..1aa2baa83959 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -325,6 +325,10 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_ref_object *ref; struct drm_hash_item *hash; struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; int ret = -EINVAL; if (base->tfile != tfile && !base->shareable) @@ -350,7 +354,7 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, return -EPERM; ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), - false, false); + &ctx); if (unlikely(ret != 0)) return ret; ref = kmalloc(sizeof(*ref), GFP_KERNEL); @@ -686,7 +690,10 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, dma_buf = prime->dma_buf; if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) { DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false + }; exp_info.ops = &tdev->ops; exp_info.size = prime->size; exp_info.flags = flags; @@ -696,7 +703,7 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, * Need to create a new dma_buf, with memory accounting. */ ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size, - false, true); + &ctx); if (unlikely(ret != 0)) { mutex_unlock(&prime->mutex); goto out_unref; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index b5ba6441489f..f1a3d55ead83 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -1058,7 +1058,7 @@ void ttm_page_alloc_fini(void) _manager = NULL; } -int ttm_pool_populate(struct ttm_tt *ttm) +int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; unsigned i; @@ -1076,7 +1076,7 @@ int ttm_pool_populate(struct ttm_tt *ttm) for (i = 0; i < ttm->num_pages; ++i) { ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - PAGE_SIZE); + PAGE_SIZE, ctx); if (unlikely(ret != 0)) { ttm_pool_unpopulate(ttm); return -ENOMEM; @@ -1113,12 +1113,13 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_pool_unpopulate); -int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt) +int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + struct ttm_operation_ctx *ctx) { unsigned i, j; int r; - r = ttm_pool_populate(&tt->ttm); + r = ttm_pool_populate(&tt->ttm, ctx); if (r) return r; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c index bda00b2ab51c..3ac53918881e 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -923,7 +923,8 @@ static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) * On success pages list will hold count number of correctly * cached pages. On failure will hold the negative return value (-ENOMEM, etc). */ -int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + struct ttm_operation_ctx *ctx) { struct ttm_tt *ttm = &ttm_dma->ttm; struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; @@ -962,7 +963,7 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) break; ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size); + pool->size, ctx); if (unlikely(ret != 0)) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; @@ -998,7 +999,7 @@ skip_huge: } ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size); + pool->size, ctx); if (unlikely(ret != 0)) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 8ebc8d3560c3..5a046a3c543a 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -261,7 +261,8 @@ void ttm_tt_unbind(struct ttm_tt *ttm) } } -int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) +int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, + struct ttm_operation_ctx *ctx) { int ret = 0; @@ -271,7 +272,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) if (ttm->state == tt_bound) return 0; - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); if (ret) return ret; |
