diff options
Diffstat (limited to 'drivers/nvdimm')
| -rw-r--r-- | drivers/nvdimm/blk.c | 8 | ||||
| -rw-r--r-- | drivers/nvdimm/btt.c | 124 | ||||
| -rw-r--r-- | drivers/nvdimm/btt_devs.c | 11 | ||||
| -rw-r--r-- | drivers/nvdimm/claim.c | 6 | ||||
| -rw-r--r-- | drivers/nvdimm/nd.h | 1 | ||||
| -rw-r--r-- | drivers/nvdimm/pfn_devs.c | 4 | ||||
| -rw-r--r-- | drivers/nvdimm/pmem.c | 29 |
7 files changed, 111 insertions, 72 deletions
diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 9faaa9694d87..f12d23c49771 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -186,7 +186,7 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) * another kernel subsystem, and we just pass it through. */ if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { - bio->bi_error = -EIO; + bio->bi_status = BLK_STS_IOERR; goto out; } @@ -205,7 +205,7 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) "io error in %s sector %lld, len %d,\n", (rw == READ) ? "READ" : "WRITE", (unsigned long long) iter.bi_sector, len); - bio->bi_error = err; + bio->bi_status = errno_to_blk_status(err); break; } } @@ -218,7 +218,8 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) } static int nsblk_rw_bytes(struct nd_namespace_common *ndns, - resource_size_t offset, void *iobuf, size_t n, int rw) + resource_size_t offset, void *iobuf, size_t n, int rw, + unsigned long flags) { struct nd_namespace_blk *nsblk = to_nd_namespace_blk(&ndns->dev); struct nd_blk_region *ndbr = to_ndbr(nsblk); @@ -272,7 +273,6 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) blk_queue_make_request(q, nd_blk_make_request); blk_queue_max_hw_sectors(q, UINT_MAX); - blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); blk_queue_logical_block_size(q, nsblk_sector_size(nsblk)); queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); q->queuedata = nsblk; diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 368795aad5c9..b6ba0618ea46 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -32,45 +32,53 @@ enum log_ent_request { }; static int arena_read_bytes(struct arena_info *arena, resource_size_t offset, - void *buf, size_t n) + void *buf, size_t n, unsigned long flags) { struct nd_btt *nd_btt = arena->nd_btt; struct nd_namespace_common *ndns = nd_btt->ndns; /* arena offsets are 4K from the base of the device */ offset += SZ_4K; - return nvdimm_read_bytes(ndns, offset, buf, n); + return nvdimm_read_bytes(ndns, offset, buf, n, flags); } static int arena_write_bytes(struct arena_info *arena, resource_size_t offset, - void *buf, size_t n) + void *buf, size_t n, unsigned long flags) { struct nd_btt *nd_btt = arena->nd_btt; struct nd_namespace_common *ndns = nd_btt->ndns; /* arena offsets are 4K from the base of the device */ offset += SZ_4K; - return nvdimm_write_bytes(ndns, offset, buf, n); + return nvdimm_write_bytes(ndns, offset, buf, n, flags); } static int btt_info_write(struct arena_info *arena, struct btt_sb *super) { int ret; + /* + * infooff and info2off should always be at least 512B aligned. + * We rely on that to make sure rw_bytes does error clearing + * correctly, so make sure that is the case. + */ + WARN_ON_ONCE(!IS_ALIGNED(arena->infooff, 512)); + WARN_ON_ONCE(!IS_ALIGNED(arena->info2off, 512)); + ret = arena_write_bytes(arena, arena->info2off, super, - sizeof(struct btt_sb)); + sizeof(struct btt_sb), 0); if (ret) return ret; return arena_write_bytes(arena, arena->infooff, super, - sizeof(struct btt_sb)); + sizeof(struct btt_sb), 0); } static int btt_info_read(struct arena_info *arena, struct btt_sb *super) { WARN_ON(!super); return arena_read_bytes(arena, arena->infooff, super, - sizeof(struct btt_sb)); + sizeof(struct btt_sb), 0); } /* @@ -79,16 +87,17 @@ static int btt_info_read(struct arena_info *arena, struct btt_sb *super) * mapping is in little-endian * mapping contains 'E' and 'Z' flags as desired */ -static int __btt_map_write(struct arena_info *arena, u32 lba, __le32 mapping) +static int __btt_map_write(struct arena_info *arena, u32 lba, __le32 mapping, + unsigned long flags) { u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE); WARN_ON(lba >= arena->external_nlba); - return arena_write_bytes(arena, ns_off, &mapping, MAP_ENT_SIZE); + return arena_write_bytes(arena, ns_off, &mapping, MAP_ENT_SIZE, flags); } static int btt_map_write(struct arena_info *arena, u32 lba, u32 mapping, - u32 z_flag, u32 e_flag) + u32 z_flag, u32 e_flag, unsigned long rwb_flags) { u32 ze; __le32 mapping_le; @@ -127,11 +136,11 @@ static int btt_map_write(struct arena_info *arena, u32 lba, u32 mapping, } mapping_le = cpu_to_le32(mapping); - return __btt_map_write(arena, lba, mapping_le); + return __btt_map_write(arena, lba, mapping_le, rwb_flags); } static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping, - int *trim, int *error) + int *trim, int *error, unsigned long rwb_flags) { int ret; __le32 in; @@ -140,7 +149,7 @@ static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping, WARN_ON(lba >= arena->external_nlba); - ret = arena_read_bytes(arena, ns_off, &in, MAP_ENT_SIZE); + ret = arena_read_bytes(arena, ns_off, &in, MAP_ENT_SIZE, rwb_flags); if (ret) return ret; @@ -189,7 +198,7 @@ static int btt_log_read_pair(struct arena_info *arena, u32 lane, WARN_ON(!ent); return arena_read_bytes(arena, arena->logoff + (2 * lane * LOG_ENT_SIZE), ent, - 2 * LOG_ENT_SIZE); + 2 * LOG_ENT_SIZE, 0); } static struct dentry *debugfs_root; @@ -335,7 +344,7 @@ static int btt_log_read(struct arena_info *arena, u32 lane, * btt_flog_write is the wrapper for updating the freelist elements */ static int __btt_log_write(struct arena_info *arena, u32 lane, - u32 sub, struct log_entry *ent) + u32 sub, struct log_entry *ent, unsigned long flags) { int ret; /* @@ -350,13 +359,13 @@ static int __btt_log_write(struct arena_info *arena, u32 lane, void *src = ent; /* split the 16B write into atomic, durable halves */ - ret = arena_write_bytes(arena, ns_off, src, log_half); + ret = arena_write_bytes(arena, ns_off, src, log_half, flags); if (ret) return ret; ns_off += log_half; src += log_half; - return arena_write_bytes(arena, ns_off, src, log_half); + return arena_write_bytes(arena, ns_off, src, log_half, flags); } static int btt_flog_write(struct arena_info *arena, u32 lane, u32 sub, @@ -364,7 +373,7 @@ static int btt_flog_write(struct arena_info *arena, u32 lane, u32 sub, { int ret; - ret = __btt_log_write(arena, lane, sub, ent); + ret = __btt_log_write(arena, lane, sub, ent, NVDIMM_IO_ATOMIC); if (ret) return ret; @@ -393,11 +402,19 @@ static int btt_map_init(struct arena_info *arena) if (!zerobuf) return -ENOMEM; + /* + * mapoff should always be at least 512B aligned. We rely on that to + * make sure rw_bytes does error clearing correctly, so make sure that + * is the case. + */ + WARN_ON_ONCE(!IS_ALIGNED(arena->mapoff, 512)); + while (mapsize) { size_t size = min(mapsize, chunk_size); + WARN_ON_ONCE(size < 512); ret = arena_write_bytes(arena, arena->mapoff + offset, zerobuf, - size); + size, 0); if (ret) goto free; @@ -417,26 +434,50 @@ static int btt_map_init(struct arena_info *arena) */ static int btt_log_init(struct arena_info *arena) { + size_t logsize = arena->info2off - arena->logoff; + size_t chunk_size = SZ_4K, offset = 0; + struct log_entry log; + void *zerobuf; int ret; u32 i; - struct log_entry log, zerolog; - memset(&zerolog, 0, sizeof(zerolog)); + zerobuf = kzalloc(chunk_size, GFP_KERNEL); + if (!zerobuf) + return -ENOMEM; + /* + * logoff should always be at least 512B aligned. We rely on that to + * make sure rw_bytes does error clearing correctly, so make sure that + * is the case. + */ + WARN_ON_ONCE(!IS_ALIGNED(arena->logoff, 512)); + + while (logsize) { + size_t size = min(logsize, chunk_size); + + WARN_ON_ONCE(size < 512); + ret = arena_write_bytes(arena, arena->logoff + offset, zerobuf, + size, 0); + if (ret) + goto free; + + offset += size; + logsize -= size; + cond_resched(); + } for (i = 0; i < arena->nfree; i++) { log.lba = cpu_to_le32(i); log.old_map = cpu_to_le32(arena->external_nlba + i); log.new_map = cpu_to_le32(arena->external_nlba + i); log.seq = cpu_to_le32(LOG_SEQ_INIT); - ret = __btt_log_write(arena, i, 0, &log); - if (ret) - return ret; - ret = __btt_log_write(arena, i, 1, &zerolog); + ret = __btt_log_write(arena, i, 0, &log, 0); if (ret) - return ret; + goto free; } - return 0; + free: + kfree(zerobuf); + return ret; } static int btt_freelist_init(struct arena_info *arena) @@ -470,7 +511,7 @@ static int btt_freelist_init(struct arena_info *arena) /* Check if map recovery is needed */ ret = btt_map_read(arena, le32_to_cpu(log_new.lba), &map_entry, - NULL, NULL); + NULL, NULL, 0); if (ret) return ret; if ((le32_to_cpu(log_new.new_map) != map_entry) && @@ -480,7 +521,7 @@ static int btt_freelist_init(struct arena_info *arena) * to complete the map write. So fix up the map. */ ret = btt_map_write(arena, le32_to_cpu(log_new.lba), - le32_to_cpu(log_new.new_map), 0, 0); + le32_to_cpu(log_new.new_map), 0, 0, 0); if (ret) return ret; } @@ -875,7 +916,7 @@ static int btt_data_read(struct arena_info *arena, struct page *page, u64 nsoff = to_namespace_offset(arena, lba); void *mem = kmap_atomic(page); - ret = arena_read_bytes(arena, nsoff, mem + off, len); + ret = arena_read_bytes(arena, nsoff, mem + off, len, NVDIMM_IO_ATOMIC); kunmap_atomic(mem); return ret; @@ -888,7 +929,7 @@ static int btt_data_write(struct arena_info *arena, u32 lba, u64 nsoff = to_namespace_offset(arena, lba); void *mem = kmap_atomic(page); - ret = arena_write_bytes(arena, nsoff, mem + off, len); + ret = arena_write_bytes(arena, nsoff, mem + off, len, NVDIMM_IO_ATOMIC); kunmap_atomic(mem); return ret; @@ -931,10 +972,12 @@ static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip, mem = kmap_atomic(bv.bv_page); if (rw) ret = arena_write_bytes(arena, meta_nsoff, - mem + bv.bv_offset, cur_len); + mem + bv.bv_offset, cur_len, + NVDIMM_IO_ATOMIC); else ret = arena_read_bytes(arena, meta_nsoff, - mem + bv.bv_offset, cur_len); + mem + bv.bv_offset, cur_len, + NVDIMM_IO_ATOMIC); kunmap_atomic(mem); if (ret) @@ -976,7 +1019,8 @@ static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip, cur_len = min(btt->sector_size, len); - ret = btt_map_read(arena, premap, &postmap, &t_flag, &e_flag); + ret = btt_map_read(arena, premap, &postmap, &t_flag, &e_flag, + NVDIMM_IO_ATOMIC); if (ret) goto out_lane; @@ -1006,7 +1050,7 @@ static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip, barrier(); ret = btt_map_read(arena, premap, &new_map, &t_flag, - &e_flag); + &e_flag, NVDIMM_IO_ATOMIC); if (ret) goto out_rtt; @@ -1093,7 +1137,8 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip, } lock_map(arena, premap); - ret = btt_map_read(arena, premap, &old_postmap, NULL, NULL); + ret = btt_map_read(arena, premap, &old_postmap, NULL, NULL, + NVDIMM_IO_ATOMIC); if (ret) goto out_map; if (old_postmap >= arena->internal_nlba) { @@ -1110,7 +1155,7 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip, if (ret) goto out_map; - ret = btt_map_write(arena, premap, new_postmap, 0, 0); + ret = btt_map_write(arena, premap, new_postmap, 0, 0, 0); if (ret) goto out_map; @@ -1165,7 +1210,7 @@ static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio) * another kernel subsystem, and we just pass it through. */ if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { - bio->bi_error = -EIO; + bio->bi_status = BLK_STS_IOERR; goto out; } @@ -1187,7 +1232,7 @@ static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio) (op_is_write(bio_op(bio))) ? "WRITE" : "READ", (unsigned long long) iter.bi_sector, len); - bio->bi_error = err; + bio->bi_status = errno_to_blk_status(err); break; } } @@ -1252,7 +1297,6 @@ static int btt_blk_init(struct btt *btt) blk_queue_make_request(btt->btt_queue, btt_make_request); blk_queue_logical_block_size(btt->btt_queue, btt->sector_size); blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX); - blk_queue_bounce_limit(btt->btt_queue, BLK_BOUNCE_ANY); queue_flag_set_unlocked(QUEUE_FLAG_NONROT, btt->btt_queue); btt->btt_queue->queuedata = btt; diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c index 4b76af2b8715..4c989bb9a8a0 100644 --- a/drivers/nvdimm/btt_devs.c +++ b/drivers/nvdimm/btt_devs.c @@ -222,13 +222,6 @@ struct device *nd_btt_create(struct nd_region *nd_region) return dev; } -static bool uuid_is_null(u8 *uuid) -{ - static const u8 null_uuid[16]; - - return (memcmp(uuid, null_uuid, 16) == 0); -} - /** * nd_btt_arena_is_valid - check if the metadata layout is valid * @nd_btt: device with BTT geometry and backing device info @@ -249,7 +242,7 @@ bool nd_btt_arena_is_valid(struct nd_btt *nd_btt, struct btt_sb *super) if (memcmp(super->signature, BTT_SIG, BTT_SIG_LEN) != 0) return false; - if (!uuid_is_null(super->parent_uuid)) + if (!guid_is_null((guid_t *)&super->parent_uuid)) if (memcmp(super->parent_uuid, parent_uuid, 16) != 0) return false; @@ -273,7 +266,7 @@ static int __nd_btt_probe(struct nd_btt *nd_btt, if (!btt_sb || !ndns || !nd_btt) return -ENODEV; - if (nvdimm_read_bytes(ndns, SZ_4K, btt_sb, sizeof(*btt_sb))) + if (nvdimm_read_bytes(ndns, SZ_4K, btt_sb, sizeof(*btt_sb), 0)) return -ENXIO; if (nvdimm_namespace_capacity(ndns) < SZ_16M) diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c index 93d128da1c92..7ceb5fa4f2a1 100644 --- a/drivers/nvdimm/claim.c +++ b/drivers/nvdimm/claim.c @@ -228,7 +228,8 @@ u64 nd_sb_checksum(struct nd_gen_sb *nd_gen_sb) EXPORT_SYMBOL(nd_sb_checksum); static int nsio_rw_bytes(struct nd_namespace_common *ndns, - resource_size_t offset, void *buf, size_t size, int rw) + resource_size_t offset, void *buf, size_t size, int rw, + unsigned long flags) { struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); unsigned int sz_align = ALIGN(size + (offset & (512 - 1)), 512); @@ -259,7 +260,8 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns, * work around this collision. */ if (IS_ALIGNED(offset, 512) && IS_ALIGNED(size, 512) - && (!ndns->claim || !is_nd_btt(ndns->claim))) { + && !(flags & NVDIMM_IO_ATOMIC) + && !ndns->claim) { long cleared; cleared = nvdimm_clear_poison(&ndns->dev, diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index 77d032192bf7..03852d738eec 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -31,6 +31,7 @@ enum { ND_MAX_LANES = 256, SECTOR_SHIFT = 9, INT_LBASIZE_ALIGNMENT = 64, + NVDIMM_IO_ATOMIC = 1, }; struct nd_poison { diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c index 335c8175410b..a6c403600d19 100644 --- a/drivers/nvdimm/pfn_devs.c +++ b/drivers/nvdimm/pfn_devs.c @@ -357,7 +357,7 @@ int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) if (!is_nd_pmem(nd_pfn->dev.parent)) return -ENODEV; - if (nvdimm_read_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb))) + if (nvdimm_read_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb), 0)) return -ENXIO; if (memcmp(pfn_sb->signature, sig, PFN_SIG_LEN) != 0) @@ -662,7 +662,7 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn) checksum = nd_sb_checksum((struct nd_gen_sb *) pfn_sb); pfn_sb->checksum = cpu_to_le64(checksum); - return nvdimm_write_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb)); + return nvdimm_write_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb), 0); } /* diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index c544d466ea51..6b577afb1d44 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -49,19 +49,19 @@ static struct nd_region *to_region(struct pmem_device *pmem) return to_nd_region(to_dev(pmem)->parent); } -static int pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, - unsigned int len) +static blk_status_t pmem_clear_poison(struct pmem_device *pmem, + phys_addr_t offset, unsigned int len) { struct device *dev = to_dev(pmem); sector_t sector; long cleared; - int rc = 0; + blk_status_t rc = BLK_STS_OK; sector = (offset - pmem->data_offset) / 512; cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len); if (cleared < len) - rc = -EIO; + rc = BLK_STS_IOERR; if (cleared > 0 && cleared / 512) { cleared /= 512; dev_dbg(dev, "%s: %#llx clear %ld sector%s\n", __func__, @@ -84,7 +84,7 @@ static void write_pmem(void *pmem_addr, struct page *page, kunmap_atomic(mem); } -static int read_pmem(struct page *page, unsigned int off, +static blk_status_t read_pmem(struct page *page, unsigned int off, void *pmem_addr, unsigned int len) { int rc; @@ -93,15 +93,15 @@ static int read_pmem(struct page *page, unsigned int off, rc = memcpy_mcsafe(mem + off, pmem_addr, len); kunmap_atomic(mem); if (rc) - return -EIO; - return 0; + return BLK_STS_IOERR; + return BLK_STS_OK; } -static int pmem_do_bvec(struct pmem_device *pmem, struct page *page, +static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page, unsigned int len, unsigned int off, bool is_write, sector_t sector) { - int rc = 0; + blk_status_t rc = BLK_STS_OK; bool bad_pmem = false; phys_addr_t pmem_off = sector * 512 + pmem->data_offset; void *pmem_addr = pmem->virt_addr + pmem_off; @@ -111,7 +111,7 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page, if (!is_write) { if (unlikely(bad_pmem)) - rc = -EIO; + rc = BLK_STS_IOERR; else { rc = read_pmem(page, off, pmem_addr, len); flush_dcache_page(page); @@ -149,7 +149,7 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page, static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) { - int rc = 0; + blk_status_t rc = 0; bool do_acct; unsigned long start; struct bio_vec bvec; @@ -166,7 +166,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) bvec.bv_offset, op_is_write(bio_op(bio)), iter.bi_sector); if (rc) { - bio->bi_error = rc; + bio->bi_status = rc; break; } } @@ -184,7 +184,7 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, struct page *page, bool is_write) { struct pmem_device *pmem = bdev->bd_queue->queuedata; - int rc; + blk_status_t rc; rc = pmem_do_bvec(pmem, page, PAGE_SIZE, 0, is_write, sector); @@ -197,7 +197,7 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, if (rc == 0) page_endio(page, is_write, 0); - return rc; + return blk_status_to_errno(rc); } /* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */ @@ -343,7 +343,6 @@ static int pmem_attach_disk(struct device *dev, blk_queue_make_request(q, pmem_make_request); blk_queue_physical_block_size(q, PAGE_SIZE); blk_queue_max_hw_sectors(q, UINT_MAX); - blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); queue_flag_set_unlocked(QUEUE_FLAG_DAX, q); q->queuedata = pmem; |
