nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

bio: add pcpu caching for non-polling bio_put

Browse source 
This patch extends REQ_ALLOC_CACHE to IRQ completions, whenever
currently it's only limited to iopoll. Instead of guarding the list with
irq toggling on alloc, which is expensive, it keeps an additional
irq-safe list from which bios are spliced in batches to ammortise
overhead. On the put side it toggles irqs, but in many cases they're
already disabled and so cheap.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c2306de96b900ab9264f4428ec37768ddcf0da36.1667384020.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-11-02 15:18:22 +00:00 committed by Jens Axboe
parent f25cf75a45
commit b99182c501
1 changed files with 54 additions and 16 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

64
block/bio.c
View file

@ -25,9 +25,15 @@
#include "blk-rq-qos.h"
#include "blk-cgroup.h"
#define ALLOC_CACHE_THRESHOLD 16
#define ALLOC_CACHE_SLACK 64
#define ALLOC_CACHE_MAX 512
struct bio_alloc_cache {
struct bio *free_list;
struct bio *free_list_irq;
unsigned int nr;
unsigned int nr_irq;
};
static struct biovec_slab {
@ -408,6 +414,22 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
queue_work(bs->rescue_workqueue, &bs->rescue_work);
}
static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache)
{
unsigned long flags;
/* cache->free_list must be empty */
if (WARN_ON_ONCE(cache->free_list))
return;
local_irq_save(flags);
cache->free_list = cache->free_list_irq;
cache->free_list_irq = NULL;
cache->nr += cache->nr_irq;
cache->nr_irq = 0;
local_irq_restore(flags);
}
static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp,
struct bio_set *bs)
@ -416,10 +438,14 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
struct bio *bio;
cache = per_cpu_ptr(bs->cache, get_cpu());
if (!cache->free_list) {
if (READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
bio_alloc_irq_cache_splice(cache);
if (!cache->free_list) {
put_cpu();
return NULL;
}
}
bio = cache->free_list;
cache->free_list = bio->bi_next;
cache->nr--;
@ -462,9 +488,6 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
* submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
* for per bio allocations.
*
* If REQ_ALLOC_CACHE is set, the final put of the bio MUST be done from process
* context, not hard/soft IRQ.
*
* Returns: Pointer to new bio on success, NULL on failure.
*/
struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
@ -678,10 +701,7 @@ void guard_bio_eod(struct bio *bio)
bio_truncate(bio, maxsector << 9);
}
#define ALLOC_CACHE_MAX 512
#define ALLOC_CACHE_SLACK 64
static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache,
unsigned int nr)
{
unsigned int i = 0;
@ -694,6 +714,17 @@ static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
if (++i == nr)
break;
}
return i;
}
static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
unsigned int nr)
{
nr -= __bio_alloc_cache_prune(cache, nr);
if (!READ_ONCE(cache->free_list)) {
bio_alloc_irq_cache_splice(cache);
__bio_alloc_cache_prune(cache, nr);
}
}
static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node)
@ -732,6 +763,12 @@ static inline void bio_put_percpu_cache(struct bio *bio)
struct bio_alloc_cache *cache;
cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX) {
put_cpu();
bio_free(bio);
return;
}
bio_uninit(bio);
if ((bio->bi_opf & REQ_POLLED) && !WARN_ON_ONCE(in_interrupt())) {
@ -739,13 +776,14 @@ static inline void bio_put_percpu_cache(struct bio *bio)
cache->free_list = bio;
cache->nr++;
} else {
put_cpu();
bio_free(bio);
return;
}
unsigned long flags;
if (cache->nr > ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK)
bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK);
local_irq_save(flags);
bio->bi_next = cache->free_list_irq;
cache->free_list_irq = bio;
cache->nr_irq++;
local_irq_restore(flags);
}
put_cpu();
}