mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm: simplify lock_page_memcg()

Browse source 
Now that migration doesn't clear page->mem_cgroup of live pages anymore,
it's safe to make lock_page_memcg() and the memcg stat functions take
pages, and spare the callers from memcg objects.

[akpm@linux-foundation.org: fix warnings]
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Suggested-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Acked-by: Vladimir Davydov <vdavydov@virtuozzo.com>
Cc: Michal Hocko <mhocko@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Johannes Weiner 2016-03-15 14:57:22 -07:00 committed by Linus Torvalds
parent 6a93ca8fde
commit 62cccb8c8e
12 changed files with 88 additions and 117 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

18
fs/buffer.c
View file

@ -624,14 +624,14 @@ EXPORT_SYMBOL(mark_buffer_dirty_inode);
* The caller must hold lock_page_memcg(). * The caller must hold lock_page_memcg().
*/ */
static void __set_page_dirty(struct page *page, struct address_space *mapping, static void __set_page_dirty(struct page *page, struct address_space *mapping,
struct mem_cgroup *memcg, int warn) int warn)
{ {
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
if (page->mapping) { /* Race with truncate? */ if (page->mapping) { /* Race with truncate? */
WARN_ON_ONCE(warn && !PageUptodate(page)); WARN_ON_ONCE(warn && !PageUptodate(page));
account_page_dirtied(page, mapping, memcg); account_page_dirtied(page, mapping);
radix_tree_tag_set(&mapping->page_tree, radix_tree_tag_set(&mapping->page_tree,
page_index(page), PAGECACHE_TAG_DIRTY); page_index(page), PAGECACHE_TAG_DIRTY);
} }
@ -666,7 +666,6 @@ static void __set_page_dirty(struct page *page, struct address_space *mapping,
int __set_page_dirty_buffers(struct page *page) int __set_page_dirty_buffers(struct page *page)
{ {
int newly_dirty; int newly_dirty;
struct mem_cgroup *memcg;
struct address_space *mapping = page_mapping(page); struct address_space *mapping = page_mapping(page);
if (unlikely(!mapping)) if (unlikely(!mapping))
@ -686,14 +685,14 @@ int __set_page_dirty_buffers(struct page *page)
* Lock out page->mem_cgroup migration to keep PageDirty * Lock out page->mem_cgroup migration to keep PageDirty
* synchronized with per-memcg dirty page counters. * synchronized with per-memcg dirty page counters.
*/ */
memcg = lock_page_memcg(page); lock_page_memcg(page);
newly_dirty = !TestSetPageDirty(page); newly_dirty = !TestSetPageDirty(page);
spin_unlock(&mapping->private_lock); spin_unlock(&mapping->private_lock);
if (newly_dirty) if (newly_dirty)
__set_page_dirty(page, mapping, memcg, 1); __set_page_dirty(page, mapping, 1);
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (newly_dirty) if (newly_dirty)
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES); __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
@ -1167,15 +1166,14 @@ void mark_buffer_dirty(struct buffer_head *bh)
if (!test_set_buffer_dirty(bh)) { if (!test_set_buffer_dirty(bh)) {
struct page *page = bh->b_page; struct page *page = bh->b_page;
struct address_space *mapping = NULL; struct address_space *mapping = NULL;
struct mem_cgroup *memcg;
memcg = lock_page_memcg(page); lock_page_memcg(page);
if (!TestSetPageDirty(page)) { if (!TestSetPageDirty(page)) {
mapping = page_mapping(page); mapping = page_mapping(page);
if (mapping) if (mapping)
__set_page_dirty(page, mapping, memcg, 0); __set_page_dirty(page, mapping, 0);
} }
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (mapping) if (mapping)
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES); __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
} }

7
fs/xfs/xfs_aops.c
View file

@ -1957,7 +1957,6 @@ xfs_vm_set_page_dirty(
loff_t end_offset; loff_t end_offset;
loff_t offset; loff_t offset;
int newly_dirty; int newly_dirty;
struct mem_cgroup *memcg;
if (unlikely(!mapping)) if (unlikely(!mapping))
return !TestSetPageDirty(page); return !TestSetPageDirty(page);
@ -1981,7 +1980,7 @@ xfs_vm_set_page_dirty(
* Lock out page->mem_cgroup migration to keep PageDirty * Lock out page->mem_cgroup migration to keep PageDirty
* synchronized with per-memcg dirty page counters. * synchronized with per-memcg dirty page counters.
*/ */
memcg = lock_page_memcg(page); lock_page_memcg(page);
newly_dirty = !TestSetPageDirty(page); newly_dirty = !TestSetPageDirty(page);
spin_unlock(&mapping->private_lock); spin_unlock(&mapping->private_lock);
@ -1992,13 +1991,13 @@ xfs_vm_set_page_dirty(
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
if (page->mapping) { /* Race with truncate? */ if (page->mapping) { /* Race with truncate? */
WARN_ON_ONCE(!PageUptodate(page)); WARN_ON_ONCE(!PageUptodate(page));
account_page_dirtied(page, mapping, memcg); account_page_dirtied(page, mapping);
radix_tree_tag_set(&mapping->page_tree, radix_tree_tag_set(&mapping->page_tree,
page_index(page), PAGECACHE_TAG_DIRTY); page_index(page), PAGECACHE_TAG_DIRTY);
} }
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
} }
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (newly_dirty) if (newly_dirty)
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES); __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
return newly_dirty; return newly_dirty;

35
include/linux/memcontrol.h
View file

@ -455,42 +455,42 @@ bool mem_cgroup_oom_synchronize(bool wait);
extern int do_swap_account; extern int do_swap_account;
#endif #endif
struct mem_cgroup *lock_page_memcg(struct page *page); void lock_page_memcg(struct page *page);
void unlock_page_memcg(struct mem_cgroup *memcg); void unlock_page_memcg(struct page *page);
/** /**
* mem_cgroup_update_page_stat - update page state statistics * mem_cgroup_update_page_stat - update page state statistics
* @memcg: memcg to account against * @page: the page
* @idx: page state item to account * @idx: page state item to account
* @val: number of pages (positive or negative) * @val: number of pages (positive or negative)
* *
* Callers must use lock_page_memcg() to prevent double accounting * Callers must use lock_page_memcg() to prevent double accounting
* when the page is concurrently being moved to another memcg: * when the page is concurrently being moved to another memcg:
* *
* memcg = lock_page_memcg(page); * lock_page_memcg(page);
* if (TestClearPageState(page)) * if (TestClearPageState(page))
* mem_cgroup_update_page_stat(memcg, state, -1); * mem_cgroup_update_page_stat(page, state, -1);
* unlock_page_memcg(memcg); * unlock_page_memcg(page);
*/ */
static inline void mem_cgroup_update_page_stat(struct mem_cgroup *memcg, static inline void mem_cgroup_update_page_stat(struct page *page,
enum mem_cgroup_stat_index idx, int val) enum mem_cgroup_stat_index idx, int val)
{ {
VM_BUG_ON(!rcu_read_lock_held()); VM_BUG_ON(!rcu_read_lock_held());
if (memcg) if (page->mem_cgroup)
this_cpu_add(memcg->stat->count[idx], val); this_cpu_add(page->mem_cgroup->stat->count[idx], val);
} }
static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg, static inline void mem_cgroup_inc_page_stat(struct page *page,
enum mem_cgroup_stat_index idx) enum mem_cgroup_stat_index idx)
{ {
mem_cgroup_update_page_stat(memcg, idx, 1); mem_cgroup_update_page_stat(page, idx, 1);
} }
static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg, static inline void mem_cgroup_dec_page_stat(struct page *page,
enum mem_cgroup_stat_index idx) enum mem_cgroup_stat_index idx)
{ {
mem_cgroup_update_page_stat(memcg, idx, -1); mem_cgroup_update_page_stat(page, idx, -1);
} }
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
@ -661,12 +661,11 @@ mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{ {
} }
static inline struct mem_cgroup *lock_page_memcg(struct page *page) static inline void lock_page_memcg(struct page *page)
{ {
return NULL;
} }
static inline void unlock_page_memcg(struct mem_cgroup *memcg) static inline void unlock_page_memcg(struct page *page)
{ {
} }
@ -692,12 +691,12 @@ static inline bool mem_cgroup_oom_synchronize(bool wait)
return false; return false;
} }
static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg, static inline void mem_cgroup_inc_page_stat(struct page *page,
enum mem_cgroup_stat_index idx) enum mem_cgroup_stat_index idx)
{ {
} }
static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg, static inline void mem_cgroup_dec_page_stat(struct page *page,
enum mem_cgroup_stat_index idx) enum mem_cgroup_stat_index idx)
{ {
} }

5
include/linux/mm.h
View file

@ -1291,10 +1291,9 @@ int __set_page_dirty_nobuffers(struct page *page);
int __set_page_dirty_no_writeback(struct page *page); int __set_page_dirty_no_writeback(struct page *page);
int redirty_page_for_writepage(struct writeback_control *wbc, int redirty_page_for_writepage(struct writeback_control *wbc,
struct page *page); struct page *page);
void account_page_dirtied(struct page *page, struct address_space *mapping, void account_page_dirtied(struct page *page, struct address_space *mapping);
struct mem_cgroup *memcg);
void account_page_cleaned(struct page *page, struct address_space *mapping, void account_page_cleaned(struct page *page, struct address_space *mapping,
struct mem_cgroup *memcg, struct bdi_writeback *wb); struct bdi_writeback *wb);
int set_page_dirty(struct page *page); int set_page_dirty(struct page *page);
int set_page_dirty_lock(struct page *page); int set_page_dirty_lock(struct page *page);
void cancel_dirty_page(struct page *page); void cancel_dirty_page(struct page *page);

3
include/linux/pagemap.h
View file

@ -663,8 +663,7 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
int add_to_page_cache_lru(struct page *page, struct address_space *mapping, int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask); pgoff_t index, gfp_t gfp_mask);
extern void delete_from_page_cache(struct page *page); extern void delete_from_page_cache(struct page *page);
extern void __delete_from_page_cache(struct page *page, void *shadow, extern void __delete_from_page_cache(struct page *page, void *shadow);
struct mem_cgroup *memcg);
int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask);
/* /*

20
mm/filemap.c
View file

@ -179,8 +179,7 @@ static void page_cache_tree_delete(struct address_space *mapping,
* is safe. The caller must hold the mapping's tree_lock and * is safe. The caller must hold the mapping's tree_lock and
* lock_page_memcg(). * lock_page_memcg().
*/ */
void __delete_from_page_cache(struct page *page, void *shadow, void __delete_from_page_cache(struct page *page, void *shadow)
struct mem_cgroup *memcg)
{ {
struct address_space *mapping = page->mapping; struct address_space *mapping = page->mapping;
@ -239,8 +238,7 @@ void __delete_from_page_cache(struct page *page, void *shadow,
* anyway will be cleared before returning page into buddy allocator. * anyway will be cleared before returning page into buddy allocator.
*/ */
if (WARN_ON_ONCE(PageDirty(page))) if (WARN_ON_ONCE(PageDirty(page)))
account_page_cleaned(page, mapping, memcg, account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
inode_to_wb(mapping->host));
} }
/** /**
@ -254,7 +252,6 @@ void __delete_from_page_cache(struct page *page, void *shadow,
void delete_from_page_cache(struct page *page) void delete_from_page_cache(struct page *page)
{ {
struct address_space *mapping = page->mapping; struct address_space *mapping = page->mapping;
struct mem_cgroup *memcg;
unsigned long flags; unsigned long flags;
void (*freepage)(struct page *); void (*freepage)(struct page *);
@ -263,11 +260,11 @@ void delete_from_page_cache(struct page *page)
freepage = mapping->a_ops->freepage; freepage = mapping->a_ops->freepage;
memcg = lock_page_memcg(page); lock_page_memcg(page);
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
__delete_from_page_cache(page, NULL, memcg); __delete_from_page_cache(page, NULL);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (freepage) if (freepage)
freepage(page); freepage(page);
@ -551,7 +548,6 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
if (!error) { if (!error) {
struct address_space *mapping = old->mapping; struct address_space *mapping = old->mapping;
void (*freepage)(struct page *); void (*freepage)(struct page *);
struct mem_cgroup *memcg;
unsigned long flags; unsigned long flags;
pgoff_t offset = old->index; pgoff_t offset = old->index;
@ -561,9 +557,9 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
new->mapping = mapping; new->mapping = mapping;
new->index = offset; new->index = offset;
memcg = lock_page_memcg(old); lock_page_memcg(old);
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
__delete_from_page_cache(old, NULL, memcg); __delete_from_page_cache(old, NULL);
error = radix_tree_insert(&mapping->page_tree, offset, new); error = radix_tree_insert(&mapping->page_tree, offset, new);
BUG_ON(error); BUG_ON(error);
mapping->nrpages++; mapping->nrpages++;
@ -576,7 +572,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
if (PageSwapBacked(new)) if (PageSwapBacked(new))
__inc_zone_page_state(new, NR_SHMEM); __inc_zone_page_state(new, NR_SHMEM);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(old);
mem_cgroup_migrate(old, new); mem_cgroup_migrate(old, new);
radix_tree_preload_end(); radix_tree_preload_end();
if (freepage) if (freepage)

23
mm/memcontrol.c
View file

@ -1690,7 +1690,7 @@ bool mem_cgroup_oom_synchronize(bool handle)
* This function protects unlocked LRU pages from being moved to * This function protects unlocked LRU pages from being moved to
* another cgroup and stabilizes their page->mem_cgroup binding. * another cgroup and stabilizes their page->mem_cgroup binding.
*/ */
struct mem_cgroup *lock_page_memcg(struct page *page) void lock_page_memcg(struct page *page)
{ {
struct mem_cgroup *memcg; struct mem_cgroup *memcg;
unsigned long flags; unsigned long flags;
@ -1699,25 +1699,18 @@ struct mem_cgroup *lock_page_memcg(struct page *page)
* The RCU lock is held throughout the transaction. The fast * The RCU lock is held throughout the transaction. The fast
* path can get away without acquiring the memcg->move_lock * path can get away without acquiring the memcg->move_lock
* because page moving starts with an RCU grace period. * because page moving starts with an RCU grace period.
*
* The RCU lock also protects the memcg from being freed when
* the page state that is going to change is the only thing
* preventing the page from being uncharged.
* E.g. end-writeback clearing PageWriteback(), which allows
* migration to go ahead and uncharge the page before the
* account transaction might be complete.
*/ */
rcu_read_lock(); rcu_read_lock();
if (mem_cgroup_disabled()) if (mem_cgroup_disabled())
return NULL; return;
again: again:
memcg = page->mem_cgroup; memcg = page->mem_cgroup;
if (unlikely(!memcg)) if (unlikely(!memcg))
return NULL; return;
if (atomic_read(&memcg->moving_account) <= 0) if (atomic_read(&memcg->moving_account) <= 0)
return memcg; return;
spin_lock_irqsave(&memcg->move_lock, flags); spin_lock_irqsave(&memcg->move_lock, flags);
if (memcg != page->mem_cgroup) { if (memcg != page->mem_cgroup) {
@ -1733,16 +1726,18 @@ struct mem_cgroup *lock_page_memcg(struct page *page)
memcg->move_lock_task = current; memcg->move_lock_task = current;
memcg->move_lock_flags = flags; memcg->move_lock_flags = flags;
return memcg; return;
} }
EXPORT_SYMBOL(lock_page_memcg); EXPORT_SYMBOL(lock_page_memcg);
/** /**
* unlock_page_memcg - unlock a page->mem_cgroup binding * unlock_page_memcg - unlock a page->mem_cgroup binding
* @memcg: the memcg returned by lock_page_memcg() * @page: the page
*/ */
void unlock_page_memcg(struct mem_cgroup *memcg) void unlock_page_memcg(struct page *page)
{ {
struct mem_cgroup *memcg = page->mem_cgroup;
if (memcg && memcg->move_lock_task == current) { if (memcg && memcg->move_lock_task == current) {
unsigned long flags = memcg->move_lock_flags; unsigned long flags = memcg->move_lock_flags;

49
mm/page-writeback.c
View file

@ -2414,8 +2414,7 @@ int __set_page_dirty_no_writeback(struct page *page)
* *
* NOTE: This relies on being atomic wrt interrupts. * NOTE: This relies on being atomic wrt interrupts.
*/ */
void account_page_dirtied(struct page *page, struct address_space *mapping, void account_page_dirtied(struct page *page, struct address_space *mapping)
struct mem_cgroup *memcg)
{ {
struct inode *inode = mapping->host; struct inode *inode = mapping->host;
@ -2427,7 +2426,7 @@ void account_page_dirtied(struct page *page, struct address_space *mapping,
inode_attach_wb(inode, page); inode_attach_wb(inode, page);
wb = inode_to_wb(inode); wb = inode_to_wb(inode);
mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_DIRTY); mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_DIRTY);
__inc_zone_page_state(page, NR_FILE_DIRTY); __inc_zone_page_state(page, NR_FILE_DIRTY);
__inc_zone_page_state(page, NR_DIRTIED); __inc_zone_page_state(page, NR_DIRTIED);
__inc_wb_stat(wb, WB_RECLAIMABLE); __inc_wb_stat(wb, WB_RECLAIMABLE);
@ -2445,10 +2444,10 @@ EXPORT_SYMBOL(account_page_dirtied);
* Caller must hold lock_page_memcg(). * Caller must hold lock_page_memcg().
*/ */
void account_page_cleaned(struct page *page, struct address_space *mapping, void account_page_cleaned(struct page *page, struct address_space *mapping,
struct mem_cgroup *memcg, struct bdi_writeback *wb) struct bdi_writeback *wb)
{ {
if (mapping_cap_account_dirty(mapping)) { if (mapping_cap_account_dirty(mapping)) {
mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_DIRTY); mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_DIRTY);
dec_zone_page_state(page, NR_FILE_DIRTY); dec_zone_page_state(page, NR_FILE_DIRTY);
dec_wb_stat(wb, WB_RECLAIMABLE); dec_wb_stat(wb, WB_RECLAIMABLE);
task_io_account_cancelled_write(PAGE_CACHE_SIZE); task_io_account_cancelled_write(PAGE_CACHE_SIZE);
@ -2469,26 +2468,24 @@ void account_page_cleaned(struct page *page, struct address_space *mapping,
*/ */
int __set_page_dirty_nobuffers(struct page *page) int __set_page_dirty_nobuffers(struct page *page)
{ {
struct mem_cgroup *memcg; lock_page_memcg(page);
memcg = lock_page_memcg(page);
if (!TestSetPageDirty(page)) { if (!TestSetPageDirty(page)) {
struct address_space *mapping = page_mapping(page); struct address_space *mapping = page_mapping(page);
unsigned long flags; unsigned long flags;
if (!mapping) { if (!mapping) {
unlock_page_memcg(memcg); unlock_page_memcg(page);
return 1; return 1;
} }
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
BUG_ON(page_mapping(page) != mapping); BUG_ON(page_mapping(page) != mapping);
WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
account_page_dirtied(page, mapping, memcg); account_page_dirtied(page, mapping);
radix_tree_tag_set(&mapping->page_tree, page_index(page), radix_tree_tag_set(&mapping->page_tree, page_index(page),
PAGECACHE_TAG_DIRTY); PAGECACHE_TAG_DIRTY);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (mapping->host) { if (mapping->host) {
/* !PageAnon && !swapper_space */ /* !PageAnon && !swapper_space */
@ -2496,7 +2493,7 @@ int __set_page_dirty_nobuffers(struct page *page)
} }
return 1; return 1;
} }
unlock_page_memcg(memcg); unlock_page_memcg(page);
return 0; return 0;
} }
EXPORT_SYMBOL(__set_page_dirty_nobuffers); EXPORT_SYMBOL(__set_page_dirty_nobuffers);
@ -2626,17 +2623,16 @@ void cancel_dirty_page(struct page *page)
if (mapping_cap_account_dirty(mapping)) { if (mapping_cap_account_dirty(mapping)) {
struct inode *inode = mapping->host; struct inode *inode = mapping->host;
struct bdi_writeback *wb; struct bdi_writeback *wb;
struct mem_cgroup *memcg;
bool locked; bool locked;
memcg = lock_page_memcg(page); lock_page_memcg(page);
wb = unlocked_inode_to_wb_begin(inode, &locked); wb = unlocked_inode_to_wb_begin(inode, &locked);
if (TestClearPageDirty(page)) if (TestClearPageDirty(page))
account_page_cleaned(page, mapping, memcg, wb); account_page_cleaned(page, mapping, wb);
unlocked_inode_to_wb_end(inode, locked); unlocked_inode_to_wb_end(inode, locked);
unlock_page_memcg(memcg); unlock_page_memcg(page);
} else { } else {
ClearPageDirty(page); ClearPageDirty(page);
} }
@ -2667,7 +2663,6 @@ int clear_page_dirty_for_io(struct page *page)
if (mapping && mapping_cap_account_dirty(mapping)) { if (mapping && mapping_cap_account_dirty(mapping)) {
struct inode *inode = mapping->host; struct inode *inode = mapping->host;
struct bdi_writeback *wb; struct bdi_writeback *wb;
struct mem_cgroup *memcg;
bool locked; bool locked;
/* /*
@ -2705,16 +2700,16 @@ int clear_page_dirty_for_io(struct page *page)
* always locked coming in here, so we get the desired * always locked coming in here, so we get the desired
* exclusion. * exclusion.
*/ */
memcg = lock_page_memcg(page); lock_page_memcg(page);
wb = unlocked_inode_to_wb_begin(inode, &locked); wb = unlocked_inode_to_wb_begin(inode, &locked);
if (TestClearPageDirty(page)) { if (TestClearPageDirty(page)) {
mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_DIRTY); mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_DIRTY);
dec_zone_page_state(page, NR_FILE_DIRTY); dec_zone_page_state(page, NR_FILE_DIRTY);
dec_wb_stat(wb, WB_RECLAIMABLE); dec_wb_stat(wb, WB_RECLAIMABLE);
ret = 1; ret = 1;
} }
unlocked_inode_to_wb_end(inode, locked); unlocked_inode_to_wb_end(inode, locked);
unlock_page_memcg(memcg); unlock_page_memcg(page);
return ret; return ret;
} }
return TestClearPageDirty(page); return TestClearPageDirty(page);
@ -2724,10 +2719,9 @@ EXPORT_SYMBOL(clear_page_dirty_for_io);
int test_clear_page_writeback(struct page *page) int test_clear_page_writeback(struct page *page)
{ {
struct address_space *mapping = page_mapping(page); struct address_space *mapping = page_mapping(page);
struct mem_cgroup *memcg;
int ret; int ret;
memcg = lock_page_memcg(page); lock_page_memcg(page);
if (mapping) { if (mapping) {
struct inode *inode = mapping->host; struct inode *inode = mapping->host;
struct backing_dev_info *bdi = inode_to_bdi(inode); struct backing_dev_info *bdi = inode_to_bdi(inode);
@ -2751,21 +2745,20 @@ int test_clear_page_writeback(struct page *page)
ret = TestClearPageWriteback(page); ret = TestClearPageWriteback(page);
} }
if (ret) { if (ret) {
mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_WRITEBACK); mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_WRITEBACK);
dec_zone_page_state(page, NR_WRITEBACK); dec_zone_page_state(page, NR_WRITEBACK);
inc_zone_page_state(page, NR_WRITTEN); inc_zone_page_state(page, NR_WRITTEN);
} }
unlock_page_memcg(memcg); unlock_page_memcg(page);
return ret; return ret;
} }
int __test_set_page_writeback(struct page *page, bool keep_write) int __test_set_page_writeback(struct page *page, bool keep_write)
{ {
struct address_space *mapping = page_mapping(page); struct address_space *mapping = page_mapping(page);
struct mem_cgroup *memcg;
int ret; int ret;
memcg = lock_page_memcg(page); lock_page_memcg(page);
if (mapping) { if (mapping) {
struct inode *inode = mapping->host; struct inode *inode = mapping->host;
struct backing_dev_info *bdi = inode_to_bdi(inode); struct backing_dev_info *bdi = inode_to_bdi(inode);
@ -2793,10 +2786,10 @@ int __test_set_page_writeback(struct page *page, bool keep_write)
ret = TestSetPageWriteback(page); ret = TestSetPageWriteback(page);
} }
if (!ret) { if (!ret) {
mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_WRITEBACK); mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_WRITEBACK);
inc_zone_page_state(page, NR_WRITEBACK); inc_zone_page_state(page, NR_WRITEBACK);
} }
unlock_page_memcg(memcg); unlock_page_memcg(page);
return ret; return ret;
} }

16
mm/rmap.c
View file

@ -1287,21 +1287,17 @@ void page_add_new_anon_rmap(struct page *page,
*/ */
void page_add_file_rmap(struct page *page) void page_add_file_rmap(struct page *page)
{ {
struct mem_cgroup *memcg; lock_page_memcg(page);
memcg = lock_page_memcg(page);
if (atomic_inc_and_test(&page->_mapcount)) { if (atomic_inc_and_test(&page->_mapcount)) {
__inc_zone_page_state(page, NR_FILE_MAPPED); __inc_zone_page_state(page, NR_FILE_MAPPED);
mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED); mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
} }
unlock_page_memcg(memcg); unlock_page_memcg(page);
} }
static void page_remove_file_rmap(struct page *page) static void page_remove_file_rmap(struct page *page)
{ {
struct mem_cgroup *memcg; lock_page_memcg(page);
memcg = lock_page_memcg(page);
/* Hugepages are not counted in NR_FILE_MAPPED for now. */ /* Hugepages are not counted in NR_FILE_MAPPED for now. */
if (unlikely(PageHuge(page))) { if (unlikely(PageHuge(page))) {
@ -1320,12 +1316,12 @@ static void page_remove_file_rmap(struct page *page)
* pte lock(a spinlock) is held, which implies preemption disabled. * pte lock(a spinlock) is held, which implies preemption disabled.
*/ */
__dec_zone_page_state(page, NR_FILE_MAPPED); __dec_zone_page_state(page, NR_FILE_MAPPED);
mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED); mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
if (unlikely(PageMlocked(page))) if (unlikely(PageMlocked(page)))
clear_page_mlock(page); clear_page_mlock(page);
out: out:
unlock_page_memcg(memcg); unlock_page_memcg(page);
} }
static void page_remove_anon_compound_rmap(struct page *page) static void page_remove_anon_compound_rmap(struct page *page)

9
mm/truncate.c
View file

@ -519,7 +519,6 @@ EXPORT_SYMBOL(invalidate_mapping_pages);
static int static int
invalidate_complete_page2(struct address_space *mapping, struct page *page) invalidate_complete_page2(struct address_space *mapping, struct page *page)
{ {
struct mem_cgroup *memcg;
unsigned long flags; unsigned long flags;
if (page->mapping != mapping) if (page->mapping != mapping)
@ -528,15 +527,15 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
return 0; return 0;
memcg = lock_page_memcg(page); lock_page_memcg(page);
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
if (PageDirty(page)) if (PageDirty(page))
goto failed; goto failed;
BUG_ON(page_has_private(page)); BUG_ON(page_has_private(page));
__delete_from_page_cache(page, NULL, memcg); __delete_from_page_cache(page, NULL);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (mapping->a_ops->freepage) if (mapping->a_ops->freepage)
mapping->a_ops->freepage(page); mapping->a_ops->freepage(page);
@ -545,7 +544,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
return 1; return 1;
failed: failed:
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
return 0; return 0;
} }

11
mm/vmscan.c
View file

@ -603,12 +603,11 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
bool reclaimed) bool reclaimed)
{ {
unsigned long flags; unsigned long flags;
struct mem_cgroup *memcg;
BUG_ON(!PageLocked(page)); BUG_ON(!PageLocked(page));
BUG_ON(mapping != page_mapping(page)); BUG_ON(mapping != page_mapping(page));
memcg = lock_page_memcg(page); lock_page_memcg(page);
spin_lock_irqsave(&mapping->tree_lock, flags); spin_lock_irqsave(&mapping->tree_lock, flags);
/* /*
* The non racy check for a busy page. * The non racy check for a busy page.
@ -648,7 +647,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
mem_cgroup_swapout(page, swap); mem_cgroup_swapout(page, swap);
__delete_from_swap_cache(page); __delete_from_swap_cache(page);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
swapcache_free(swap); swapcache_free(swap);
} else { } else {
void (*freepage)(struct page *); void (*freepage)(struct page *);
@ -674,9 +673,9 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
if (reclaimed && page_is_file_cache(page) && if (reclaimed && page_is_file_cache(page) &&
!mapping_exiting(mapping) && !dax_mapping(mapping)) !mapping_exiting(mapping) && !dax_mapping(mapping))
shadow = workingset_eviction(mapping, page); shadow = workingset_eviction(mapping, page);
__delete_from_page_cache(page, shadow, memcg); __delete_from_page_cache(page, shadow);
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
if (freepage != NULL) if (freepage != NULL)
freepage(page); freepage(page);
@ -686,7 +685,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
cannot_free: cannot_free:
spin_unlock_irqrestore(&mapping->tree_lock, flags); spin_unlock_irqrestore(&mapping->tree_lock, flags);
unlock_page_memcg(memcg); unlock_page_memcg(page);
return 0; return 0;
} }

9
mm/workingset.c
View file

@ -305,10 +305,9 @@ bool workingset_refault(void *shadow)
*/ */
void workingset_activation(struct page *page) void workingset_activation(struct page *page)
{ {
struct mem_cgroup *memcg;
struct lruvec *lruvec; struct lruvec *lruvec;
memcg = lock_page_memcg(page); lock_page_memcg(page);
/* /*
* Filter non-memcg pages here, e.g. unmap can call * Filter non-memcg pages here, e.g. unmap can call
* mark_page_accessed() on VDSO pages. * mark_page_accessed() on VDSO pages.
@ -316,12 +315,12 @@ void workingset_activation(struct page *page)
* XXX: See workingset_refault() - this should return * XXX: See workingset_refault() - this should return
* root_mem_cgroup even for !CONFIG_MEMCG. * root_mem_cgroup even for !CONFIG_MEMCG.
*/ */
if (!mem_cgroup_disabled() && !memcg) if (!mem_cgroup_disabled() && !page_memcg(page))
goto out; goto out;
lruvec = mem_cgroup_zone_lruvec(page_zone(page), memcg); lruvec = mem_cgroup_zone_lruvec(page_zone(page), page_memcg(page));
atomic_long_inc(&lruvec->inactive_age); atomic_long_inc(&lruvec->inactive_age);
out: out:
unlock_page_memcg(memcg); unlock_page_memcg(page);
} }
/* /*