nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

block: allow bio_for_each_segment_all() to iterate over multi-page bvec

Browse source 
This patch introduces one extra iterator variable to bio_for_each_segment_all(),
then we can allow bio_for_each_segment_all() to iterate over multi-page bvec.

Given it is just one mechannical & simple change on all bio_for_each_segment_all()
users, this patch does tree-wide change in one single patch, so that we can
avoid to use a temporary helper for this conversion.

Reviewed-by: Omar Sandoval <osandov@fb.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Ming Lei 2019-02-15 19:13:19 +08:00 committed by Jens Axboe
parent 2e1f4f4d24
commit 6dc4f100c1
27 changed files with 127 additions and 46 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

27
block/bio.c
View file

@ -1072,8 +1072,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
{ {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
ssize_t ret; ssize_t ret;
ret = copy_page_from_iter(bvec->bv_page, ret = copy_page_from_iter(bvec->bv_page,
@ -1103,8 +1104,9 @@ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
{ {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
ssize_t ret; ssize_t ret;
ret = copy_page_to_iter(bvec->bv_page, ret = copy_page_to_iter(bvec->bv_page,
@ -1126,8 +1128,9 @@ void bio_free_pages(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
__free_page(bvec->bv_page); __free_page(bvec->bv_page);
} }
EXPORT_SYMBOL(bio_free_pages); EXPORT_SYMBOL(bio_free_pages);
@ -1295,6 +1298,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
struct bio *bio; struct bio *bio;
int ret; int ret;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
if (!iov_iter_count(iter)) if (!iov_iter_count(iter))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
@ -1368,7 +1372,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
return bio; return bio;
out_unmap: out_unmap:
bio_for_each_segment_all(bvec, bio, j) { bio_for_each_segment_all(bvec, bio, j, iter_all) {
put_page(bvec->bv_page); put_page(bvec->bv_page);
} }
bio_put(bio); bio_put(bio);
@ -1379,11 +1383,12 @@ static void __bio_unmap_user(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
/* /*
* make sure we dirty pages we wrote to * make sure we dirty pages we wrote to
*/ */
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
if (bio_data_dir(bio) == READ) if (bio_data_dir(bio) == READ)
set_page_dirty_lock(bvec->bv_page); set_page_dirty_lock(bvec->bv_page);
@ -1475,8 +1480,9 @@ static void bio_copy_kern_endio_read(struct bio *bio)
char *p = bio->bi_private; char *p = bio->bi_private;
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
memcpy(p, page_address(bvec->bv_page), bvec->bv_len); memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
p += bvec->bv_len; p += bvec->bv_len;
} }
@ -1585,8 +1591,9 @@ void bio_set_pages_dirty(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
if (!PageCompound(bvec->bv_page)) if (!PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page); set_page_dirty_lock(bvec->bv_page);
} }
@ -1596,8 +1603,9 @@ static void bio_release_pages(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
put_page(bvec->bv_page); put_page(bvec->bv_page);
} }
@ -1644,8 +1652,9 @@ void bio_check_pages_dirty(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
unsigned long flags; unsigned long flags;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
goto defer; goto defer;
} }

6
block/bounce.c
View file

@ -165,11 +165,12 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool)
struct bio_vec *bvec, orig_vec; struct bio_vec *bvec, orig_vec;
int i; int i;
struct bvec_iter orig_iter = bio_orig->bi_iter; struct bvec_iter orig_iter = bio_orig->bi_iter;
struct bvec_iter_all iter_all;
/* /*
* free up bounce indirect pages used * free up bounce indirect pages used
*/ */
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
orig_vec = bio_iter_iovec(bio_orig, orig_iter); orig_vec = bio_iter_iovec(bio_orig, orig_iter);
if (bvec->bv_page != orig_vec.bv_page) { if (bvec->bv_page != orig_vec.bv_page) {
dec_zone_page_state(bvec->bv_page, NR_BOUNCE); dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
@ -294,6 +295,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
bool bounce = false; bool bounce = false;
int sectors = 0; int sectors = 0;
bool passthrough = bio_is_passthrough(*bio_orig); bool passthrough = bio_is_passthrough(*bio_orig);
struct bvec_iter_all iter_all;
bio_for_each_segment(from, *bio_orig, iter) { bio_for_each_segment(from, *bio_orig, iter) {
if (i++ < BIO_MAX_PAGES) if (i++ < BIO_MAX_PAGES)
@ -313,7 +315,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL : bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL :
&bounce_bio_set); &bounce_bio_set);
bio_for_each_segment_all(to, bio, i) { bio_for_each_segment_all(to, bio, i, iter_all) {
struct page *page = to->bv_page; struct page *page = to->bv_page;
if (page_to_pfn(page) <= q->limits.bounce_pfn) if (page_to_pfn(page) <= q->limits.bounce_pfn)

3
drivers/md/bcache/btree.c
View file

@ -432,8 +432,9 @@ static void do_btree_node_write(struct btree *b)
int j; int j;
struct bio_vec *bv; struct bio_vec *bv;
void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1)); void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1));
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, b->bio, j) bio_for_each_segment_all(bv, b->bio, j, iter_all)
memcpy(page_address(bv->bv_page), memcpy(page_address(bv->bv_page),
base + j * PAGE_SIZE, PAGE_SIZE); base + j * PAGE_SIZE, PAGE_SIZE);

3
drivers/md/dm-crypt.c
View file

@ -1447,8 +1447,9 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
{ {
unsigned int i; unsigned int i;
struct bio_vec *bv; struct bio_vec *bv;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, clone, i) { bio_for_each_segment_all(bv, clone, i, iter_all) {
BUG_ON(!bv->bv_page); BUG_ON(!bv->bv_page);
mempool_free(bv->bv_page, &cc->page_pool); mempool_free(bv->bv_page, &cc->page_pool);
} }

3
drivers/md/raid1.c
View file

@ -2112,13 +2112,14 @@ static void process_checks(struct r1bio *r1_bio)
struct page **spages = get_resync_pages(sbio)->pages; struct page **spages = get_resync_pages(sbio)->pages;
struct bio_vec *bi; struct bio_vec *bi;
int page_len[RESYNC_PAGES] = { 0 }; int page_len[RESYNC_PAGES] = { 0 };
struct bvec_iter_all iter_all;
if (sbio->bi_end_io != end_sync_read) if (sbio->bi_end_io != end_sync_read)
continue; continue;
/* Now we can 'fixup' the error value */ /* Now we can 'fixup' the error value */
sbio->bi_status = 0; sbio->bi_status = 0;
bio_for_each_segment_all(bi, sbio, j) bio_for_each_segment_all(bi, sbio, j, iter_all)
page_len[j] = bi->bv_len; page_len[j] = bi->bv_len;
if (!status) { if (!status) {

3
drivers/staging/erofs/data.c
View file

@ -20,8 +20,9 @@ static inline void read_endio(struct bio *bio)
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
const blk_status_t err = bio->bi_status; const blk_status_t err = bio->bi_status;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
/* page is already locked */ /* page is already locked */

3
drivers/staging/erofs/unzip_vle.c
View file

@ -830,8 +830,9 @@ static inline void z_erofs_vle_read_endio(struct bio *bio)
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
struct address_space *mc = NULL; struct address_space *mc = NULL;
#endif #endif
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
bool cachemngd = false; bool cachemngd = false;

6
fs/block_dev.c
View file

@ -211,6 +211,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
ssize_t ret; ssize_t ret;
blk_qc_t qc; blk_qc_t qc;
int i; int i;
struct bvec_iter_all iter_all;
if ((pos | iov_iter_alignment(iter)) & if ((pos | iov_iter_alignment(iter)) &
(bdev_logical_block_size(bdev) - 1)) (bdev_logical_block_size(bdev) - 1))
@ -260,7 +261,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
} }
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
bio_for_each_segment_all(bvec, &bio, i) { bio_for_each_segment_all(bvec, &bio, i, iter_all) {
if (should_dirty && !PageCompound(bvec->bv_page)) if (should_dirty && !PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page); set_page_dirty_lock(bvec->bv_page);
put_page(bvec->bv_page); put_page(bvec->bv_page);
@ -329,8 +330,9 @@ static void blkdev_bio_end_io(struct bio *bio)
} else { } else {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
put_page(bvec->bv_page); put_page(bvec->bv_page);
bio_put(bio); bio_put(bio);
} }

3
fs/btrfs/compression.c
View file

@ -162,13 +162,14 @@ static void end_compressed_bio_read(struct bio *bio)
} else { } else {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
/* /*
* we have verified the checksum already, set page * we have verified the checksum already, set page
* checked so the end_io handlers know about it * checked so the end_io handlers know about it
*/ */
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, cb->orig_bio, i) bio_for_each_segment_all(bvec, cb->orig_bio, i, iter_all)
SetPageChecked(bvec->bv_page); SetPageChecked(bvec->bv_page);
bio_endio(cb->orig_bio); bio_endio(cb->orig_bio);

3
fs/btrfs/disk-io.c
View file

@ -832,9 +832,10 @@ static blk_status_t btree_csum_one_bio(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct btrfs_root *root; struct btrfs_root *root;
int i, ret = 0; int i, ret = 0;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
root = BTRFS_I(bvec->bv_page->mapping->host)->root; root = BTRFS_I(bvec->bv_page->mapping->host)->root;
ret = csum_dirty_buffer(root->fs_info, bvec->bv_page); ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
if (ret) if (ret)

9
fs/btrfs/extent_io.c
View file

@ -2422,9 +2422,10 @@ static void end_bio_extent_writepage(struct bio *bio)
u64 start; u64 start;
u64 end; u64 end;
int i; int i;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host; struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@ -2493,9 +2494,10 @@ static void end_bio_extent_readpage(struct bio *bio)
int mirror; int mirror;
int ret; int ret;
int i; int i;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host; struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@ -3635,9 +3637,10 @@ static void end_bio_extent_buffer_writepage(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct extent_buffer *eb; struct extent_buffer *eb;
int i, done; int i, done;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
eb = (struct extent_buffer *)page->private; eb = (struct extent_buffer *)page->private;

6
fs/btrfs/inode.c
View file

@ -7777,6 +7777,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct extent_io_tree *io_tree, *failure_tree; struct extent_io_tree *io_tree, *failure_tree;
int i; int i;
struct bvec_iter_all iter_all;
if (bio->bi_status) if (bio->bi_status)
goto end; goto end;
@ -7788,7 +7789,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
done->uptodate = 1; done->uptodate = 1;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
io_tree, done->start, bvec->bv_page, io_tree, done->start, bvec->bv_page,
btrfs_ino(BTRFS_I(inode)), 0); btrfs_ino(BTRFS_I(inode)), 0);
@ -7867,6 +7868,7 @@ static void btrfs_retry_endio(struct bio *bio)
int uptodate; int uptodate;
int ret; int ret;
int i; int i;
struct bvec_iter_all iter_all;
if (bio->bi_status) if (bio->bi_status)
goto end; goto end;
@ -7880,7 +7882,7 @@ static void btrfs_retry_endio(struct bio *bio)
failure_tree = &BTRFS_I(inode)->io_failure_tree; failure_tree = &BTRFS_I(inode)->io_failure_tree;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
bvec->bv_offset, done->start, bvec->bv_offset, done->start,
bvec->bv_len); bvec->bv_len);

3
fs/btrfs/raid56.c
View file

@ -1443,10 +1443,11 @@ static void set_bio_pages_uptodate(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
ASSERT(!bio_flagged(bio, BIO_CLONED)); ASSERT(!bio_flagged(bio, BIO_CLONED));
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
SetPageUptodate(bvec->bv_page); SetPageUptodate(bvec->bv_page);
} }

3
fs/crypto/bio.c
View file

@ -30,8 +30,9 @@ static void __fscrypt_decrypt_bio(struct bio *bio, bool done)
{ {
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
int ret = fscrypt_decrypt_page(page->mapping->host, page, int ret = fscrypt_decrypt_page(page->mapping->host, page,
PAGE_SIZE, 0, page->index); PAGE_SIZE, 0, page->index);

4
fs/direct-io.c
View file

@ -551,7 +551,9 @@ static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) { if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) {
bio_check_pages_dirty(bio); /* transfers ownership */ bio_check_pages_dirty(bio); /* transfers ownership */
} else { } else {
bio_for_each_segment_all(bvec, bio, i) { struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
if (dio->op == REQ_OP_READ && !PageCompound(page) && if (dio->op == REQ_OP_READ && !PageCompound(page) &&

3
fs/exofs/ore.c
View file

@ -420,8 +420,9 @@ static void _clear_bio(struct bio *bio)
{ {
struct bio_vec *bv; struct bio_vec *bv;
unsigned i; unsigned i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
unsigned this_count = bv->bv_len; unsigned this_count = bv->bv_len;
if (likely(PAGE_SIZE == this_count)) if (likely(PAGE_SIZE == this_count))

3
fs/exofs/ore_raid.c
View file

@ -468,11 +468,12 @@ static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
/* loop on all devices all pages */ /* loop on all devices all pages */
for (d = 0; d < ios->numdevs; d++) { for (d = 0; d < ios->numdevs; d++) {
struct bio *bio = ios->per_dev[d].bio; struct bio *bio = ios->per_dev[d].bio;
struct bvec_iter_all iter_all;
if (!bio) if (!bio)
continue; continue;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
SetPageUptodate(page); SetPageUptodate(page);

3
fs/ext4/page-io.c
View file

@ -63,8 +63,9 @@ static void ext4_finish_bio(struct bio *bio)
{ {
int i; int i;
struct bio_vec *bvec; struct bio_vec *bvec;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
#ifdef CONFIG_EXT4_FS_ENCRYPTION #ifdef CONFIG_EXT4_FS_ENCRYPTION
struct page *data_page = NULL; struct page *data_page = NULL;

3
fs/ext4/readpage.c
View file

@ -72,6 +72,7 @@ static void mpage_end_io(struct bio *bio)
{ {
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
if (ext4_bio_encrypted(bio)) { if (ext4_bio_encrypted(bio)) {
if (bio->bi_status) { if (bio->bi_status) {
@ -81,7 +82,7 @@ static void mpage_end_io(struct bio *bio)
return; return;
} }
} }
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
if (!bio->bi_status) { if (!bio->bi_status) {

9
fs/f2fs/data.c
View file

@ -87,8 +87,9 @@ static void __read_end_io(struct bio *bio)
struct page *page; struct page *page;
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
page = bv->bv_page; page = bv->bv_page;
/* PG_error was set if any post_read step failed */ /* PG_error was set if any post_read step failed */
@ -164,13 +165,14 @@ static void f2fs_write_end_io(struct bio *bio)
struct f2fs_sb_info *sbi = bio->bi_private; struct f2fs_sb_info *sbi = bio->bi_private;
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
if (time_to_inject(sbi, FAULT_WRITE_IO)) { if (time_to_inject(sbi, FAULT_WRITE_IO)) {
f2fs_show_injection_info(FAULT_WRITE_IO); f2fs_show_injection_info(FAULT_WRITE_IO);
bio->bi_status = BLK_STS_IOERR; bio->bi_status = BLK_STS_IOERR;
} }
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
enum count_type type = WB_DATA_TYPE(page); enum count_type type = WB_DATA_TYPE(page);
@ -347,6 +349,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
struct bio_vec *bvec; struct bio_vec *bvec;
struct page *target; struct page *target;
int i; int i;
struct bvec_iter_all iter_all;
if (!io->bio) if (!io->bio)
return false; return false;
@ -354,7 +357,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
if (!inode && !page && !ino) if (!inode && !page && !ino)
return true; return true;
bio_for_each_segment_all(bvec, io->bio, i) { bio_for_each_segment_all(bvec, io->bio, i, iter_all) {
if (bvec->bv_page->mapping) if (bvec->bv_page->mapping)
target = bvec->bv_page; target = bvec->bv_page;

9
fs/gfs2/lops.c
View file

@ -170,7 +170,8 @@ u64 gfs2_log_bmap(struct gfs2_sbd *sdp)
* that is pinned in the pagecache. * that is pinned in the pagecache.
*/ */
static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct bio_vec *bvec, static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,
struct bio_vec *bvec,
blk_status_t error) blk_status_t error)
{ {
struct buffer_head *bh, *next; struct buffer_head *bh, *next;
@ -208,6 +209,7 @@ static void gfs2_end_log_write(struct bio *bio)
struct bio_vec *bvec; struct bio_vec *bvec;
struct page *page; struct page *page;
int i; int i;
struct bvec_iter_all iter_all;
if (bio->bi_status) { if (bio->bi_status) {
fs_err(sdp, "Error %d writing to journal, jid=%u\n", fs_err(sdp, "Error %d writing to journal, jid=%u\n",
@ -215,7 +217,7 @@ static void gfs2_end_log_write(struct bio *bio)
wake_up(&sdp->sd_logd_waitq); wake_up(&sdp->sd_logd_waitq);
} }
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
page = bvec->bv_page; page = bvec->bv_page;
if (page_has_buffers(page)) if (page_has_buffers(page))
gfs2_end_log_write_bh(sdp, bvec, bio->bi_status); gfs2_end_log_write_bh(sdp, bvec, bio->bi_status);
@ -388,8 +390,9 @@ static void gfs2_end_log_read(struct bio *bio)
struct page *page; struct page *page;
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
page = bvec->bv_page; page = bvec->bv_page;
if (bio->bi_status) { if (bio->bi_status) {
int err = blk_status_to_errno(bio->bi_status); int err = blk_status_to_errno(bio->bi_status);

3
fs/gfs2/meta_io.c
View file

@ -190,8 +190,9 @@ static void gfs2_meta_read_endio(struct bio *bio)
{ {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) { bio_for_each_segment_all(bvec, bio, i, iter_all) {
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
struct buffer_head *bh = page_buffers(page); struct buffer_head *bh = page_buffers(page);
unsigned int len = bvec->bv_len; unsigned int len = bvec->bv_len;

6
fs/iomap.c
View file

@ -267,8 +267,9 @@ iomap_read_end_io(struct bio *bio)
int error = blk_status_to_errno(bio->bi_status); int error = blk_status_to_errno(bio->bi_status);
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
iomap_read_page_end_io(bvec, error); iomap_read_page_end_io(bvec, error);
bio_put(bio); bio_put(bio);
} }
@ -1559,8 +1560,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)
} else { } else {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
put_page(bvec->bv_page); put_page(bvec->bv_page);
bio_put(bio); bio_put(bio);
} }

3
fs/mpage.c
View file

@ -48,8 +48,9 @@ static void mpage_end_io(struct bio *bio)
{ {
struct bio_vec *bv; struct bio_vec *bv;
int i; int i;
struct bvec_iter_all iter_all;
bio_for_each_segment_all(bv, bio, i) { bio_for_each_segment_all(bv, bio, i, iter_all) {
struct page *page = bv->bv_page; struct page *page = bv->bv_page;
page_endio(page, bio_op(bio), page_endio(page, bio_op(bio),
blk_status_to_errno(bio->bi_status)); blk_status_to_errno(bio->bi_status));

5
fs/xfs/xfs_aops.c
View file

@ -62,7 +62,7 @@ xfs_find_daxdev_for_inode(
static void static void
xfs_finish_page_writeback( xfs_finish_page_writeback(
struct inode *inode, struct inode *inode,
struct bio_vec *bvec, struct bio_vec *bvec,
int error) int error)
{ {
struct iomap_page *iop = to_iomap_page(bvec->bv_page); struct iomap_page *iop = to_iomap_page(bvec->bv_page);
@ -98,6 +98,7 @@ xfs_destroy_ioend(
for (bio = &ioend->io_inline_bio; bio; bio = next) { for (bio = &ioend->io_inline_bio; bio; bio = next) {
struct bio_vec *bvec; struct bio_vec *bvec;
int i; int i;
struct bvec_iter_all iter_all;
/* /*
* For the last bio, bi_private points to the ioend, so we * For the last bio, bi_private points to the ioend, so we
@ -109,7 +110,7 @@ xfs_destroy_ioend(
next = bio->bi_private; next = bio->bi_private;
/* walk each page on bio, ending page IO on them */ /* walk each page on bio, ending page IO on them */
bio_for_each_segment_all(bvec, bio, i) bio_for_each_segment_all(bvec, bio, i, iter_all)
xfs_finish_page_writeback(inode, bvec, error); xfs_finish_page_writeback(inode, bvec, error);
bio_put(bio); bio_put(bio);
} }

11
include/linux/bio.h
View file

@ -128,12 +128,19 @@ static inline bool bio_full(struct bio *bio)
return bio->bi_vcnt >= bio->bi_max_vecs; return bio->bi_vcnt >= bio->bi_max_vecs;
} }
#define mp_bvec_for_each_segment(bv, bvl, i, iter_all) \
for (bv = bvec_init_iter_all(&iter_all); \
(iter_all.done < (bvl)->bv_len) && \
(mp_bvec_next_segment((bvl), &iter_all), 1); \
iter_all.done += bv->bv_len, i += 1)
/* /*
* drivers should _never_ use the all version - the bio may have been split * drivers should _never_ use the all version - the bio may have been split
* before it got to the driver and the driver won't own all of it * before it got to the driver and the driver won't own all of it
*/ */
#define bio_for_each_segment_all(bvl, bio, i) \ #define bio_for_each_segment_all(bvl, bio, i, iter_all) \
for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++) for (i = 0, iter_all.idx = 0; iter_all.idx < (bio)->bi_vcnt; iter_all.idx++) \
mp_bvec_for_each_segment(bvl, &((bio)->bi_io_vec[iter_all.idx]), i, iter_all)
static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
unsigned bytes) unsigned bytes)

30
include/linux/bvec.h
View file

@ -45,6 +45,12 @@ struct bvec_iter {
current bvec */ current bvec */
}; };
struct bvec_iter_all {
struct bio_vec bv;
int idx;
unsigned done;
};
/* /*
* various member access, note that bio_data should of course not be used * various member access, note that bio_data should of course not be used
* on highmem page vectors * on highmem page vectors
@ -131,6 +137,30 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
.bi_bvec_done = 0, \ .bi_bvec_done = 0, \
} }
static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
{
iter_all->bv.bv_page = NULL;
iter_all->done = 0;
return &iter_all->bv;
}
static inline void mp_bvec_next_segment(const struct bio_vec *bvec,
struct bvec_iter_all *iter_all)
{
struct bio_vec *bv = &iter_all->bv;
if (bv->bv_page) {
bv->bv_page = nth_page(bv->bv_page, 1);
bv->bv_offset = 0;
} else {
bv->bv_page = bvec->bv_page;
bv->bv_offset = bvec->bv_offset;
}
bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
bvec->bv_len - iter_all->done);
}
/* /*
* Get the last single-page segment from the multi-page bvec and store it * Get the last single-page segment from the multi-page bvec and store it
* in @seg * in @seg