nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

block: optimise for_each_bvec() advance

Browse source 
Because of how for_each_bvec() works it never advances across multiple
entries at a time, so bvec_iter_advance() is an overkill. Add
specialised bvec_iter_advance_single() that is faster. It also handles
zero-len bvecs, so can kill bvec_iter_skip_zero_bvec().

   text    data     bss     dec     hex filename
before:
  23977     805       0   24782    60ce lib/iov_iter.o
before, bvec_iter_advance() w/o WARN_ONCE()
  22886     600       0   23486    5bbe ./lib/iov_iter.o
after:
  21862     600       0   22462    57be lib/iov_iter.o

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2020-11-24 17:58:12 +00:00 committed by Jens Axboe
parent 977115c0f6
commit 6b6667aa4d
1 changed files with 15 additions and 5 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

20
include/linux/bvec.h
View file

@ -121,18 +121,28 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
return true; return true;
} }
static inline void bvec_iter_skip_zero_bvec(struct bvec_iter *iter) /*
* A simpler version of bvec_iter_advance(), @bytes should not span
* across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len
*/
static inline void bvec_iter_advance_single(const struct bio_vec *bv,
struct bvec_iter *iter, unsigned int bytes)
{ {
iter->bi_bvec_done = 0; unsigned int done = iter->bi_bvec_done + bytes;
iter->bi_idx++;
if (done == bv[iter->bi_idx].bv_len) {
done = 0;
iter->bi_idx++;
}
iter->bi_bvec_done = done;
iter->bi_size -= bytes;
} }
#define for_each_bvec(bvl, bio_vec, iter, start) \ #define for_each_bvec(bvl, bio_vec, iter, start) \
for (iter = (start); \ for (iter = (start); \
(iter).bi_size && \ (iter).bi_size && \
((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \ ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
(bvl).bv_len ? (void)bvec_iter_advance((bio_vec), &(iter), \ bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
(bvl).bv_len) : bvec_iter_skip_zero_bvec(&(iter)))
/* for iterating one bio from start to end */ /* for iterating one bio from start to end */
#define BVEC_ITER_ALL_INIT (struct bvec_iter) \ #define BVEC_ITER_ALL_INIT (struct bvec_iter) \