nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

mm: page_frag: use initial zero offset for page_frag_alloc_align()

Browse source 
We are about to use page_frag_alloc_*() API to not just
allocate memory for skb->data, but also use them to do
the memory allocation for skb frag too. Currently the
implementation of page_frag in mm subsystem is running
the offset as a countdown rather than count-up value,
there may have several advantages to that as mentioned
in [1], but it may have some disadvantages, for example,
it may disable skb frag coalescing and more correct cache
prefetching

We have a trade-off to make in order to have a unified
implementation and API for page_frag, so use a initial zero
offset in this patch, and the following patch will try to
make some optimization to avoid the disadvantages as much
as possible.

1. https://lore.kernel.org/all/f4abe71b3439b39d17a6fb2d410180f367cadf5c.camel@gmail.com/

CC: Andrew Morton <akpm@linux-foundation.org>
CC: Linux-MM <linux-mm@kvack.org>
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Reviewed-by: Alexander Duyck <alexanderduyck@fb.com>
Link: https://patch.msgid.link/20241028115343.3405838-4-linyunsheng@huawei.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Yunsheng Lin 2024-10-28 19:53:38 +08:00 committed by Jakub Kicinski
parent 65941f10ca
commit 8218f62c9c
1 changed files with 27 additions and 27 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

54
mm/page_frag_cache.c
View file

@ -63,9 +63,13 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask, unsigned int fragsz, gfp_t gfp_mask,
unsigned int align_mask) unsigned int align_mask)
{ {
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
unsigned int size = nc->size;
#else
unsigned int size = PAGE_SIZE; unsigned int size = PAGE_SIZE;
#endif
unsigned int offset;
struct page *page; struct page *page;
int offset;
if (unlikely(!nc->va)) { if (unlikely(!nc->va)) {
refill: refill:
@ -85,32 +89,12 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
/* reset page count bias and offset to start of new frag */ /* reset page count bias and offset to start of new frag */
nc->pfmemalloc = page_is_pfmemalloc(page); nc->pfmemalloc = page_is_pfmemalloc(page);
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
nc->offset = size; nc->offset = 0;
} }
offset = nc->offset - fragsz; offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask);
if (unlikely(offset < 0)) { if (unlikely(offset + fragsz > size)) {
page = virt_to_page(nc->va); if (unlikely(fragsz > PAGE_SIZE)) {
if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
goto refill;
if (unlikely(nc->pfmemalloc)) {
free_unref_page(page, compound_order(page));
goto refill;
}
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
/* if size can vary use size else just use PAGE_SIZE */
size = nc->size;
#endif
/* OK, page count is 0, we can safely set it */
set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
/* reset page count bias and offset to start of new frag */
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
offset = size - fragsz;
if (unlikely(offset < 0)) {
/* /*
* The caller is trying to allocate a fragment * The caller is trying to allocate a fragment
* with fragsz > PAGE_SIZE but the cache isn't big * with fragsz > PAGE_SIZE but the cache isn't big
@ -122,11 +106,27 @@ void *__page_frag_alloc_align(struct page_frag_cache *nc,
*/ */
return NULL; return NULL;
} }
page = virt_to_page(nc->va);
if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
goto refill;
if (unlikely(nc->pfmemalloc)) {
free_unref_page(page, compound_order(page));
goto refill;
}
/* OK, page count is 0, we can safely set it */
set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
/* reset page count bias and offset to start of new frag */
nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
offset = 0;
} }
nc->pagecnt_bias--; nc->pagecnt_bias--;
offset &= align_mask; nc->offset = offset + fragsz;
nc->offset = offset;
return nc->va + offset; return nc->va + offset;
} }