diff options
Diffstat (limited to 'block/bio.c')
| -rw-r--r-- | block/bio.c | 114 |
1 files changed, 78 insertions, 36 deletions
diff --git a/block/bio.c b/block/bio.c index 299a0e7651ec..94d697217887 100644 --- a/block/bio.c +++ b/block/bio.c @@ -233,6 +233,9 @@ fallback: void bio_uninit(struct bio *bio) { bio_disassociate_blkg(bio); + + if (bio_integrity(bio)) + bio_integrity_free(bio); } EXPORT_SYMBOL(bio_uninit); @@ -536,6 +539,55 @@ void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start) EXPORT_SYMBOL(zero_fill_bio_iter); /** + * bio_truncate - truncate the bio to small size of @new_size + * @bio: the bio to be truncated + * @new_size: new size for truncating the bio + * + * Description: + * Truncate the bio to new size of @new_size. If bio_op(bio) is + * REQ_OP_READ, zero the truncated part. This function should only + * be used for handling corner cases, such as bio eod. + */ +void bio_truncate(struct bio *bio, unsigned new_size) +{ + struct bio_vec bv; + struct bvec_iter iter; + unsigned int done = 0; + bool truncated = false; + + if (new_size >= bio->bi_iter.bi_size) + return; + + if (bio_op(bio) != REQ_OP_READ) + goto exit; + + bio_for_each_segment(bv, bio, iter) { + if (done + bv.bv_len > new_size) { + unsigned offset; + + if (!truncated) + offset = new_size - done; + else + offset = 0; + zero_user(bv.bv_page, offset, bv.bv_len - offset); + truncated = true; + } + done += bv.bv_len; + } + + exit: + /* + * Don't touch bvec table here and make it really immutable, since + * fs bio user has to retrieve all pages via bio_for_each_segment_all + * in its .end_bio() callback. + * + * It is enough to truncate bio by updating .bi_size since we can make + * correct bvec with the updated .bi_size for drivers. + */ + bio->bi_iter.bi_size = new_size; +} + +/** * bio_put - release a reference to a bio * @bio: bio to release reference to * @@ -646,25 +698,20 @@ static inline bool page_is_mergeable(const struct bio_vec *bv, return true; } -/* - * Check if the @page can be added to the current segment(@bv), and make - * sure to call it only if page_is_mergeable(@bv, @page) is true - */ -static bool can_add_page_to_seg(struct request_queue *q, - struct bio_vec *bv, struct page *page, unsigned len, - unsigned offset) +static bool bio_try_merge_pc_page(struct request_queue *q, struct bio *bio, + struct page *page, unsigned len, unsigned offset, + bool *same_page) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; unsigned long mask = queue_segment_boundary(q); phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset; phys_addr_t addr2 = page_to_phys(page) + offset + len - 1; if ((addr1 | mask) != (addr2 | mask)) return false; - if (bv->bv_len + len > queue_max_segment_size(q)) return false; - - return true; + return __bio_try_merge_page(bio, page, len, offset, same_page); } /** @@ -674,7 +721,7 @@ static bool can_add_page_to_seg(struct request_queue *q, * @page: page to add * @len: vec entry length * @offset: vec entry offset - * @put_same_page: put the page if it is same with last added page + * @same_page: return if the merge happen inside the same page * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block device @@ -685,10 +732,9 @@ static bool can_add_page_to_seg(struct request_queue *q, */ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, - bool put_same_page) + bool *same_page) { struct bio_vec *bvec; - bool same_page = false; /* * cloned bio must not modify vec list @@ -700,28 +746,16 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio, return 0; if (bio->bi_vcnt > 0) { - bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; - - if (page == bvec->bv_page && - offset == bvec->bv_offset + bvec->bv_len) { - if (put_same_page) - put_page(page); - bvec->bv_len += len; - goto done; - } + if (bio_try_merge_pc_page(q, bio, page, len, offset, same_page)) + return len; /* - * If the queue doesn't support SG gaps and adding this - * offset would create a gap, disallow it. + * If the queue doesn't support SG gaps and adding this segment + * would create a gap, disallow it. */ + bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (bvec_gap_to_prev(q, bvec, offset)) return 0; - - if (page_is_mergeable(bvec, page, len, offset, &same_page) && - can_add_page_to_seg(q, bvec, page, len, offset)) { - bvec->bv_len += len; - goto done; - } } if (bio_full(bio, len)) @@ -735,7 +769,6 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio, bvec->bv_len = len; bvec->bv_offset = offset; bio->bi_vcnt++; - done: bio->bi_iter.bi_size += len; return len; } @@ -743,7 +776,8 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio, int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { - return __bio_add_pc_page(q, bio, page, len, offset, false); + bool same_page = false; + return __bio_add_pc_page(q, bio, page, len, offset, &same_page); } EXPORT_SYMBOL(bio_add_pc_page); @@ -773,6 +807,8 @@ bool __bio_try_merge_page(struct bio *bio, struct page *page, struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (page_is_mergeable(bv, page, len, off, same_page)) { + if (bio->bi_iter.bi_size > UINT_MAX - len) + return false; bv->bv_len += len; bio->bi_iter.bi_size += len; return true; @@ -806,6 +842,9 @@ void __bio_add_page(struct bio *bio, struct page *page, bio->bi_iter.bi_size += len; bio->bi_vcnt++; + + if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) + bio_set_flag(bio, BIO_WORKINGSET); } EXPORT_SYMBOL_GPL(__bio_add_page); @@ -1384,13 +1423,17 @@ struct bio *bio_map_user_iov(struct request_queue *q, for (j = 0; j < npages; j++) { struct page *page = pages[j]; unsigned int n = PAGE_SIZE - offs; + bool same_page = false; if (n > bytes) n = bytes; if (!__bio_add_pc_page(q, bio, page, n, offs, - true)) + &same_page)) { + if (same_page) + put_page(page); break; + } added += n; bytes -= n; @@ -1521,7 +1564,6 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len, bio->bi_end_io = bio_map_kern_endio; return bio; } -EXPORT_SYMBOL(bio_map_kern); static void bio_copy_kern_endio(struct bio *bio) { @@ -1842,8 +1884,8 @@ EXPORT_SYMBOL(bio_endio); * @bio, and updates @bio to represent the remaining sectors. * * Unless this is a discard request the newly allocated bio will point - * to @bio's bi_io_vec; it is the caller's responsibility to ensure that - * @bio is not freed before the split. + * to @bio's bi_io_vec. It is the caller's responsibility to ensure that + * neither @bio nor @bs are freed before the split bio. */ struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs) |
