nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

powerpc/mm/radix: Improve TLB/PWC flushes

Browse source 
At the moment we have to rather sub-optimal flushing behaviours:

 - flush_tlb_mm() will flush the PWC which is unnecessary (for example
   when doing a fork)

 - A large unmap will call flush_tlb_pwc() multiple times causing us
   to perform that fairly expensive operation repeatedly. This happens
   often in batches of 3 on every new process.

So we change flush_tlb_mm() to only flush the TLB, and we use the
existing "need_flush_all" flag in struct mmu_gather to indicate
that the PWC needs flushing.

Unfortunately, flush_tlb_range() still needs to do a full flush
for now as it's used by the THP collapsing. We will fix that later.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
This commit is contained in:
Benjamin Herrenschmidt 2017-07-19 14:49:05 +10:00 committed by Michael Ellerman
parent 5ce5fe14ed
commit a46cc7a90f
2 changed files with 37 additions and 51 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
View file

@ -22,22 +22,20 @@ extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
int psize);
extern void radix__tlb_flush(struct mmu_gather *tlb);
#ifdef CONFIG_SMP
extern void radix__flush_tlb_mm(struct mm_struct *mm);
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
int psize);
#else
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
#define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
#define radix__flush_tlb_pwc(tlb, addr) radix__local_flush_tlb_pwc(tlb, addr)
#endif
extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
unsigned long page_size);
extern void radix__flush_tlb_lpid(unsigned long lpid);

84
arch/powerpc/mm/tlb-radix.c
View file

@ -68,17 +68,6 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
}
static inline void tlbiel_pwc(unsigned long pid)
{
asm volatile("ptesync": : :"memory");
/* For PWC flush, we don't look at set number */
__tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
asm volatile("ptesync": : :"memory");
asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
}
static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
{
unsigned long rb,rs,prs,r;
@ -149,31 +138,23 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
preempt_disable();
pid = mm->context.id;
if (pid != MMU_NO_CONTEXT)
_tlbiel_pid(pid, RIC_FLUSH_ALL);
_tlbiel_pid(pid, RIC_FLUSH_TLB);
preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_mm);
void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
#ifndef CONFIG_SMP
static void radix__local_flush_all_mm(struct mm_struct *mm)
{
unsigned long pid;
struct mm_struct *mm = tlb->mm;
/*
* If we are doing a full mm flush, we will do a tlb flush
* with RIC_FLUSH_ALL later.
*/
if (tlb->fullmm)
return;
preempt_disable();
pid = mm->context.id;
if (pid != MMU_NO_CONTEXT)
tlbiel_pwc(pid);
_tlbiel_pid(pid, RIC_FLUSH_ALL);
preempt_enable();
}
EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
#endif /* CONFIG_SMP */
void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
int psize)
@ -205,6 +186,24 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
{
unsigned long pid;
preempt_disable();
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
goto no_context;
if (!mm_is_thread_local(mm))
_tlbie_pid(pid, RIC_FLUSH_TLB);
else
_tlbiel_pid(pid, RIC_FLUSH_TLB);
no_context:
preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_mm);
static void radix__flush_all_mm(struct mm_struct *mm)
{
unsigned long pid;
preempt_disable();
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
@ -217,31 +216,10 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
no_context:
preempt_enable();
}
EXPORT_SYMBOL(radix__flush_tlb_mm);
void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
{
unsigned long pid;
struct mm_struct *mm = tlb->mm;
/*
* If we are doing a full mm flush, we will do a tlb flush
* with RIC_FLUSH_ALL later.
*/
if (tlb->fullmm)
return;
preempt_disable();
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
goto no_context;
if (!mm_is_thread_local(mm))
_tlbie_pid(pid, RIC_FLUSH_PWC);
else
tlbiel_pwc(pid);
no_context:
preempt_enable();
tlb->need_flush_all = 1;
}
EXPORT_SYMBOL(radix__flush_tlb_pwc);
@ -274,6 +252,8 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
}
EXPORT_SYMBOL(radix__flush_tlb_page);
#else /* CONFIG_SMP */
#define radix__flush_all_mm radix__local_flush_all_mm
#endif /* CONFIG_SMP */
void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
@ -291,7 +271,12 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
{
struct mm_struct *mm = vma->vm_mm;
radix__flush_tlb_mm(mm);
/*
* This is currently used when collapsing THPs so we need to
* flush the PWC. We should fix this.
*/
radix__flush_all_mm(mm);
}
EXPORT_SYMBOL(radix__flush_tlb_range);
@ -322,7 +307,10 @@ void radix__tlb_flush(struct mmu_gather *tlb)
*/
if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
else
else if (tlb->need_flush_all) {
tlb->need_flush_all = 0;
radix__flush_all_mm(mm);
} else
radix__flush_tlb_mm(mm);
}