mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm, page_alloc: check multiple page fields with a single branch

Browse source 
Every page allocated or freed is checked for sanity to avoid corruptions
that are difficult to detect later.  A bad page could be due to a number
of fields.  Instead of using multiple branches, this patch combines
multiple fields into a single branch.  A detailed check is only
necessary if that check fails.

                                             4.6.0-rc2                  4.6.0-rc2
                                        initonce-v1r20            multcheck-v1r20
  Min      alloc-odr0-1               359.00 (  0.00%)           348.00 (  3.06%)
  Min      alloc-odr0-2               260.00 (  0.00%)           254.00 (  2.31%)
  Min      alloc-odr0-4               214.00 (  0.00%)           213.00 (  0.47%)
  Min      alloc-odr0-8               186.00 (  0.00%)           186.00 (  0.00%)
  Min      alloc-odr0-16              173.00 (  0.00%)           173.00 (  0.00%)
  Min      alloc-odr0-32              165.00 (  0.00%)           166.00 ( -0.61%)
  Min      alloc-odr0-64              162.00 (  0.00%)           162.00 (  0.00%)
  Min      alloc-odr0-128             161.00 (  0.00%)           160.00 (  0.62%)
  Min      alloc-odr0-256             170.00 (  0.00%)           169.00 (  0.59%)
  Min      alloc-odr0-512             181.00 (  0.00%)           180.00 (  0.55%)
  Min      alloc-odr0-1024            190.00 (  0.00%)           188.00 (  1.05%)
  Min      alloc-odr0-2048            196.00 (  0.00%)           194.00 (  1.02%)
  Min      alloc-odr0-4096            202.00 (  0.00%)           199.00 (  1.49%)
  Min      alloc-odr0-8192            205.00 (  0.00%)           202.00 (  1.46%)
  Min      alloc-odr0-16384           205.00 (  0.00%)           203.00 (  0.98%)

Again, the benefit is marginal but avoiding excessive branches is
important.  Ideally the paths would not have to check these conditions
at all but regrettably abandoning the tests would make use-after-free
bugs much harder to detect.

Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Mel Gorman 2016-05-19 17:14:15 -07:00 committed by Linus Torvalds
parent 93ea9964d1
commit 7bfec6f47b
1 changed files with 43 additions and 12 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

51
mm/page_alloc.c
View file

@ -784,10 +784,42 @@ static inline void __free_one_page(struct page *page,
zone->free_area[order].nr_free++; zone->free_area[order].nr_free++;
} }
/*
* A bad page could be due to a number of fields. Instead of multiple branches,
* try and check multiple fields with one check. The caller must do a detailed
* check if necessary.
*/
static inline bool page_expected_state(struct page *page,
unsigned long check_flags)
{
if (unlikely(atomic_read(&page->_mapcount) != -1))
return false;
if (unlikely((unsigned long)page->mapping |
page_ref_count(page) |
#ifdef CONFIG_MEMCG
(unsigned long)page->mem_cgroup |
#endif
(page->flags & check_flags)))
return false;
return true;
}
static inline int free_pages_check(struct page *page) static inline int free_pages_check(struct page *page)
{ {
const char *bad_reason = NULL; const char *bad_reason;
unsigned long bad_flags = 0; unsigned long bad_flags;
if (page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)) {
page_cpupid_reset_last(page);
page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
return 0;
}
/* Something has gone sideways, find it */
bad_reason = NULL;
bad_flags = 0;
if (unlikely(atomic_read(&page->_mapcount) != -1)) if (unlikely(atomic_read(&page->_mapcount) != -1))
bad_reason = "nonzero mapcount"; bad_reason = "nonzero mapcount";
@ -803,14 +835,8 @@ static inline int free_pages_check(struct page *page)
if (unlikely(page->mem_cgroup)) if (unlikely(page->mem_cgroup))
bad_reason = "page still charged to cgroup"; bad_reason = "page still charged to cgroup";
#endif #endif
if (unlikely(bad_reason)) {
bad_page(page, bad_reason, bad_flags); bad_page(page, bad_reason, bad_flags);
return 1; return 1;
}
page_cpupid_reset_last(page);
if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
return 0;
} }
/* /*
@ -1492,9 +1518,14 @@ static inline void expand(struct zone *zone, struct page *page,
*/ */
static inline int check_new_page(struct page *page) static inline int check_new_page(struct page *page)
{ {
const char *bad_reason = NULL; const char *bad_reason;
unsigned long bad_flags = 0; unsigned long bad_flags;
if (page_expected_state(page, PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))
return 0;
bad_reason = NULL;
bad_flags = 0;
if (unlikely(atomic_read(&page->_mapcount) != -1)) if (unlikely(atomic_read(&page->_mapcount) != -1))
bad_reason = "nonzero mapcount"; bad_reason = "nonzero mapcount";
if (unlikely(page->mapping != NULL)) if (unlikely(page->mapping != NULL))