forked from mirrors/linux
f7b0797d36
The struct page_counter has explicit padding for better cache alignment.
The commit c6f53ed8f2 ("mm, memcg: cg2 memory{.swap,}.peak write
handlers") added a field to the struct page_counter and accidently
increased its size. Let's move the failcnt field which is v1-only field
to the same cacheline of usage to reduce the size of struct page_counter.
Link: https://lkml.kernel.org/r/20250228075808.207484-4-shakeel.butt@linux.dev
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Roman Gushchin (Cruise) <roman.gushchin@linux.dev>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
112 lines
3.3 KiB
C
112 lines
3.3 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_PAGE_COUNTER_H
|
|
#define _LINUX_PAGE_COUNTER_H
|
|
|
|
#include <linux/atomic.h>
|
|
#include <linux/cache.h>
|
|
#include <linux/limits.h>
|
|
#include <asm/page.h>
|
|
|
|
struct page_counter {
|
|
/*
|
|
* Make sure 'usage' does not share cacheline with any other field in
|
|
* v2. The memcg->memory.usage is a hot member of struct mem_cgroup.
|
|
*/
|
|
atomic_long_t usage;
|
|
unsigned long failcnt; /* v1-only field */
|
|
|
|
CACHELINE_PADDING(_pad1_);
|
|
|
|
/* effective memory.min and memory.min usage tracking */
|
|
unsigned long emin;
|
|
atomic_long_t min_usage;
|
|
atomic_long_t children_min_usage;
|
|
|
|
/* effective memory.low and memory.low usage tracking */
|
|
unsigned long elow;
|
|
atomic_long_t low_usage;
|
|
atomic_long_t children_low_usage;
|
|
|
|
unsigned long watermark;
|
|
/* Latest cg2 reset watermark */
|
|
unsigned long local_watermark;
|
|
|
|
/* Keep all the read most fields in a separete cacheline. */
|
|
CACHELINE_PADDING(_pad2_);
|
|
|
|
bool protection_support;
|
|
bool track_failcnt;
|
|
unsigned long min;
|
|
unsigned long low;
|
|
unsigned long high;
|
|
unsigned long max;
|
|
struct page_counter *parent;
|
|
} ____cacheline_internodealigned_in_smp;
|
|
|
|
#if BITS_PER_LONG == 32
|
|
#define PAGE_COUNTER_MAX LONG_MAX
|
|
#else
|
|
#define PAGE_COUNTER_MAX (LONG_MAX / PAGE_SIZE)
|
|
#endif
|
|
|
|
/*
|
|
* Protection is supported only for the first counter (with id 0).
|
|
*/
|
|
static inline void page_counter_init(struct page_counter *counter,
|
|
struct page_counter *parent,
|
|
bool protection_support)
|
|
{
|
|
counter->usage = (atomic_long_t)ATOMIC_LONG_INIT(0);
|
|
counter->max = PAGE_COUNTER_MAX;
|
|
counter->parent = parent;
|
|
counter->protection_support = protection_support;
|
|
counter->track_failcnt = false;
|
|
}
|
|
|
|
static inline unsigned long page_counter_read(struct page_counter *counter)
|
|
{
|
|
return atomic_long_read(&counter->usage);
|
|
}
|
|
|
|
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages);
|
|
void page_counter_charge(struct page_counter *counter, unsigned long nr_pages);
|
|
bool page_counter_try_charge(struct page_counter *counter,
|
|
unsigned long nr_pages,
|
|
struct page_counter **fail);
|
|
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages);
|
|
void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages);
|
|
void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages);
|
|
|
|
static inline void page_counter_set_high(struct page_counter *counter,
|
|
unsigned long nr_pages)
|
|
{
|
|
WRITE_ONCE(counter->high, nr_pages);
|
|
}
|
|
|
|
int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages);
|
|
int page_counter_memparse(const char *buf, const char *max,
|
|
unsigned long *nr_pages);
|
|
|
|
static inline void page_counter_reset_watermark(struct page_counter *counter)
|
|
{
|
|
unsigned long usage = page_counter_read(counter);
|
|
|
|
/*
|
|
* Update local_watermark first, so it's always <= watermark
|
|
* (modulo CPU/compiler re-ordering)
|
|
*/
|
|
counter->local_watermark = usage;
|
|
counter->watermark = usage;
|
|
}
|
|
|
|
#if IS_ENABLED(CONFIG_MEMCG) || IS_ENABLED(CONFIG_CGROUP_DMEM)
|
|
void page_counter_calculate_protection(struct page_counter *root,
|
|
struct page_counter *counter,
|
|
bool recursive_protection);
|
|
#else
|
|
static inline void page_counter_calculate_protection(struct page_counter *root,
|
|
struct page_counter *counter,
|
|
bool recursive_protection) {}
|
|
#endif
|
|
|
|
#endif /* _LINUX_PAGE_COUNTER_H */
|