nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

sparc64: implement the new page table range API

Browse source 
Add set_ptes(), update_mmu_cache_range(), flush_dcache_folio() and
flush_icache_pages().  Convert the PG_dcache_dirty flag from being
per-page to per-folio.

Link: https://lkml.kernel.org/r/20230802151406.3735276-27-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
Cc: "David S. Miller" <davem@davemloft.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Matthew Wilcox (Oracle) 2023-08-02 16:13:54 +01:00 committed by Andrew Morton
parent 665f640294
commit 1a10a44dfc
5 changed files with 119 additions and 67 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

18
arch/sparc/include/asm/cacheflush_64.h
View file

@ -35,20 +35,26 @@ void flush_icache_range(unsigned long start, unsigned long end);
void __flush_icache_page(unsigned long); void __flush_icache_page(unsigned long);
void __flush_dcache_page(void *addr, int flush_icache); void __flush_dcache_page(void *addr, int flush_icache);
void flush_dcache_page_impl(struct page *page); void flush_dcache_folio_impl(struct folio *folio);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
void smp_flush_dcache_page_impl(struct page *page, int cpu); void smp_flush_dcache_folio_impl(struct folio *folio, int cpu);
void flush_dcache_page_all(struct mm_struct *mm, struct page *page); void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio);
#else #else
#define smp_flush_dcache_page_impl(page,cpu) flush_dcache_page_impl(page) #define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio)
#define flush_dcache_page_all(mm,page) flush_dcache_page_impl(page) #define flush_dcache_folio_all(mm, folio) flush_dcache_folio_impl(folio)
#endif #endif
void __flush_dcache_range(unsigned long start, unsigned long end); void __flush_dcache_range(unsigned long start, unsigned long end);
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
void flush_dcache_page(struct page *page); void flush_dcache_folio(struct folio *folio);
#define flush_dcache_folio flush_dcache_folio
static inline void flush_dcache_page(struct page *page)
{
flush_dcache_folio(page_folio(page));
}
#define flush_icache_page(vma, pg) do { } while(0) #define flush_icache_page(vma, pg) do { } while(0)
#define flush_icache_pages(vma, pg, nr) do { } while(0)
void flush_ptrace_access(struct vm_area_struct *, struct page *, void flush_ptrace_access(struct vm_area_struct *, struct page *,
unsigned long uaddr, void *kaddr, unsigned long uaddr, void *kaddr,

29
arch/sparc/include/asm/pgtable_64.h
View file

@ -86,6 +86,7 @@ extern unsigned long VMALLOC_END;
#define vmemmap ((struct page *)VMEMMAP_BASE) #define vmemmap ((struct page *)VMEMMAP_BASE)
#include <linux/sched.h> #include <linux/sched.h>
#include <asm/tlbflush.h>
bool kern_addr_valid(unsigned long addr); bool kern_addr_valid(unsigned long addr);
@ -927,8 +928,21 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT); maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT);
} }
#define set_pte_at(mm,addr,ptep,pte) \ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
__set_pte_at((mm), (addr), (ptep), (pte), 0) pte_t *ptep, pte_t pte, unsigned int nr)
{
arch_enter_lazy_mmu_mode();
for (;;) {
__set_pte_at(mm, addr, ptep, pte, 0);
if (--nr == 0)
break;
ptep++;
pte_val(pte) += PAGE_SIZE;
addr += PAGE_SIZE;
}
arch_leave_lazy_mmu_mode();
}
#define set_ptes set_ptes
#define pte_clear(mm,addr,ptep) \ #define pte_clear(mm,addr,ptep) \
set_pte_at((mm), (addr), (ptep), __pte(0UL)) set_pte_at((mm), (addr), (ptep), __pte(0UL))
@ -947,8 +961,8 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
\ \
if (pfn_valid(this_pfn) && \ if (pfn_valid(this_pfn) && \
(((old_addr) ^ (new_addr)) & (1 << 13))) \ (((old_addr) ^ (new_addr)) & (1 << 13))) \
flush_dcache_page_all(current->mm, \ flush_dcache_folio_all(current->mm, \
pfn_to_page(this_pfn)); \ page_folio(pfn_to_page(this_pfn))); \
} \ } \
newpte; \ newpte; \
}) })
@ -963,7 +977,10 @@ struct seq_file;
void mmu_info(struct seq_file *); void mmu_info(struct seq_file *);
struct vm_area_struct; struct vm_area_struct;
void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); void update_mmu_cache_range(struct vm_fault *, struct vm_area_struct *,
unsigned long addr, pte_t *ptep, unsigned int nr);
#define update_mmu_cache(vma, addr, ptep) \
update_mmu_cache_range(NULL, vma, addr, ptep, 1)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
pmd_t *pmd); pmd_t *pmd);
@ -1121,8 +1138,6 @@ static inline bool pte_access_permitted(pte_t pte, bool write)
} }
#define pte_access_permitted pte_access_permitted #define pte_access_permitted pte_access_permitted
#include <asm/tlbflush.h>
/* We provide our own get_unmapped_area to cope with VA holes and /* We provide our own get_unmapped_area to cope with VA holes and
* SHM area cache aliasing for userland. * SHM area cache aliasing for userland.
*/ */

44
arch/sparc/kernel/smp_64.c
View file

@ -921,20 +921,26 @@ extern unsigned long xcall_flush_dcache_page_cheetah;
#endif #endif
extern unsigned long xcall_flush_dcache_page_spitfire; extern unsigned long xcall_flush_dcache_page_spitfire;
static inline void __local_flush_dcache_page(struct page *page) static inline void __local_flush_dcache_folio(struct folio *folio)
{ {
unsigned int i, nr = folio_nr_pages(folio);
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
__flush_dcache_page(page_address(page), for (i = 0; i < nr; i++)
__flush_dcache_page(folio_address(folio) + i * PAGE_SIZE,
((tlb_type == spitfire) && ((tlb_type == spitfire) &&
page_mapping_file(page) != NULL)); folio_flush_mapping(folio) != NULL));
#else #else
if (page_mapping_file(page) != NULL && if (folio_flush_mapping(folio) != NULL &&
tlb_type == spitfire) tlb_type == spitfire) {
__flush_icache_page(__pa(page_address(page))); unsigned long pfn = folio_pfn(folio)
for (i = 0; i < nr; i++)
__flush_icache_page((pfn + i) * PAGE_SIZE);
}
#endif #endif
} }
void smp_flush_dcache_page_impl(struct page *page, int cpu) void smp_flush_dcache_folio_impl(struct folio *folio, int cpu)
{ {
int this_cpu; int this_cpu;
@ -948,14 +954,14 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu)
this_cpu = get_cpu(); this_cpu = get_cpu();
if (cpu == this_cpu) { if (cpu == this_cpu) {
__local_flush_dcache_page(page); __local_flush_dcache_folio(folio);
} else if (cpu_online(cpu)) { } else if (cpu_online(cpu)) {
void *pg_addr = page_address(page); void *pg_addr = folio_address(folio);
u64 data0 = 0; u64 data0 = 0;
if (tlb_type == spitfire) { if (tlb_type == spitfire) {
data0 = ((u64)&xcall_flush_dcache_page_spitfire); data0 = ((u64)&xcall_flush_dcache_page_spitfire);
if (page_mapping_file(page) != NULL) if (folio_flush_mapping(folio) != NULL)
data0 |= ((u64)1 << 32); data0 |= ((u64)1 << 32);
} else if (tlb_type == cheetah || tlb_type == cheetah_plus) { } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
@ -963,18 +969,23 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu)
#endif #endif
} }
if (data0) { if (data0) {
unsigned int i, nr = folio_nr_pages(folio);
for (i = 0; i < nr; i++) {
xcall_deliver(data0, __pa(pg_addr), xcall_deliver(data0, __pa(pg_addr),
(u64) pg_addr, cpumask_of(cpu)); (u64) pg_addr, cpumask_of(cpu));
#ifdef CONFIG_DEBUG_DCFLUSH #ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes_xcall); atomic_inc(&dcpage_flushes_xcall);
#endif #endif
pg_addr += PAGE_SIZE;
}
} }
} }
put_cpu(); put_cpu();
} }
void flush_dcache_page_all(struct mm_struct *mm, struct page *page) void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
{ {
void *pg_addr; void *pg_addr;
u64 data0; u64 data0;
@ -988,10 +999,10 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
atomic_inc(&dcpage_flushes); atomic_inc(&dcpage_flushes);
#endif #endif
data0 = 0; data0 = 0;
pg_addr = page_address(page); pg_addr = folio_address(folio);
if (tlb_type == spitfire) { if (tlb_type == spitfire) {
data0 = ((u64)&xcall_flush_dcache_page_spitfire); data0 = ((u64)&xcall_flush_dcache_page_spitfire);
if (page_mapping_file(page) != NULL) if (folio_flush_mapping(folio) != NULL)
data0 |= ((u64)1 << 32); data0 |= ((u64)1 << 32);
} else if (tlb_type == cheetah || tlb_type == cheetah_plus) { } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
@ -999,13 +1010,18 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
#endif #endif
} }
if (data0) { if (data0) {
unsigned int i, nr = folio_nr_pages(folio);
for (i = 0; i < nr; i++) {
xcall_deliver(data0, __pa(pg_addr), xcall_deliver(data0, __pa(pg_addr),
(u64) pg_addr, cpu_online_mask); (u64) pg_addr, cpu_online_mask);
#ifdef CONFIG_DEBUG_DCFLUSH #ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes_xcall); atomic_inc(&dcpage_flushes_xcall);
#endif #endif
pg_addr += PAGE_SIZE;
} }
__local_flush_dcache_page(page); }
__local_flush_dcache_folio(folio);
preempt_enable(); preempt_enable();
} }

72
arch/sparc/mm/init_64.c
View file

@ -195,21 +195,26 @@ atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
#endif #endif
#endif #endif
inline void flush_dcache_page_impl(struct page *page) inline void flush_dcache_folio_impl(struct folio *folio)
{ {
unsigned int i, nr = folio_nr_pages(folio);
BUG_ON(tlb_type == hypervisor); BUG_ON(tlb_type == hypervisor);
#ifdef CONFIG_DEBUG_DCFLUSH #ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes); atomic_inc(&dcpage_flushes);
#endif #endif
#ifdef DCACHE_ALIASING_POSSIBLE #ifdef DCACHE_ALIASING_POSSIBLE
__flush_dcache_page(page_address(page), for (i = 0; i < nr; i++)
__flush_dcache_page(folio_address(folio) + i * PAGE_SIZE,
((tlb_type == spitfire) && ((tlb_type == spitfire) &&
page_mapping_file(page) != NULL)); folio_flush_mapping(folio) != NULL));
#else #else
if (page_mapping_file(page) != NULL && if (folio_flush_mapping(folio) != NULL &&
tlb_type == spitfire) tlb_type == spitfire) {
__flush_icache_page(__pa(page_address(page))); for (i = 0; i < nr; i++)
__flush_icache_page((pfn + i) * PAGE_SIZE);
}
#endif #endif
} }
@ -218,10 +223,10 @@ inline void flush_dcache_page_impl(struct page *page)
#define PG_dcache_cpu_mask \ #define PG_dcache_cpu_mask \
((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL) ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
#define dcache_dirty_cpu(page) \ #define dcache_dirty_cpu(folio) \
(((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask) (((folio)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
static inline void set_dcache_dirty(struct page *page, int this_cpu) static inline void set_dcache_dirty(struct folio *folio, int this_cpu)
{ {
unsigned long mask = this_cpu; unsigned long mask = this_cpu;
unsigned long non_cpu_bits; unsigned long non_cpu_bits;
@ -238,11 +243,11 @@ static inline void set_dcache_dirty(struct page *page, int this_cpu)
"bne,pn %%xcc, 1b\n\t" "bne,pn %%xcc, 1b\n\t"
" nop" " nop"
: /* no outputs */ : /* no outputs */
: "r" (mask), "r" (non_cpu_bits), "r" (&page->flags) : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags)
: "g1", "g7"); : "g1", "g7");
} }
static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu) static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu)
{ {
unsigned long mask = (1UL << PG_dcache_dirty); unsigned long mask = (1UL << PG_dcache_dirty);
@ -260,7 +265,7 @@ static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
" nop\n" " nop\n"
"2:" "2:"
: /* no outputs */ : /* no outputs */
: "r" (cpu), "r" (mask), "r" (&page->flags), : "r" (cpu), "r" (mask), "r" (&folio->flags),
"i" (PG_dcache_cpu_mask), "i" (PG_dcache_cpu_mask),
"i" (PG_dcache_cpu_shift) "i" (PG_dcache_cpu_shift)
: "g1", "g7"); : "g1", "g7");
@ -284,9 +289,10 @@ static void flush_dcache(unsigned long pfn)
page = pfn_to_page(pfn); page = pfn_to_page(pfn);
if (page) { if (page) {
struct folio *folio = page_folio(page);
unsigned long pg_flags; unsigned long pg_flags;
pg_flags = page->flags; pg_flags = folio->flags;
if (pg_flags & (1UL << PG_dcache_dirty)) { if (pg_flags & (1UL << PG_dcache_dirty)) {
int cpu = ((pg_flags >> PG_dcache_cpu_shift) & int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
PG_dcache_cpu_mask); PG_dcache_cpu_mask);
@ -296,11 +302,11 @@ static void flush_dcache(unsigned long pfn)
* in the SMP case. * in the SMP case.
*/ */
if (cpu == this_cpu) if (cpu == this_cpu)
flush_dcache_page_impl(page); flush_dcache_folio_impl(folio);
else else
smp_flush_dcache_page_impl(page, cpu); smp_flush_dcache_folio_impl(folio, cpu);
clear_dcache_dirty_cpu(page, cpu); clear_dcache_dirty_cpu(folio, cpu);
put_cpu(); put_cpu();
} }
@ -388,12 +394,14 @@ bool __init arch_hugetlb_valid_size(unsigned long size)
} }
#endif /* CONFIG_HUGETLB_PAGE */ #endif /* CONFIG_HUGETLB_PAGE */
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma,
unsigned long address, pte_t *ptep, unsigned int nr)
{ {
struct mm_struct *mm; struct mm_struct *mm;
unsigned long flags; unsigned long flags;
bool is_huge_tsb; bool is_huge_tsb;
pte_t pte = *ptep; pte_t pte = *ptep;
unsigned int i;
if (tlb_type != hypervisor) { if (tlb_type != hypervisor) {
unsigned long pfn = pte_pfn(pte); unsigned long pfn = pte_pfn(pte);
@ -440,15 +448,21 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
} }
} }
#endif #endif
if (!is_huge_tsb) if (!is_huge_tsb) {
for (i = 0; i < nr; i++) {
__update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
address, pte_val(pte)); address, pte_val(pte));
address += PAGE_SIZE;
pte_val(pte) += PAGE_SIZE;
}
}
spin_unlock_irqrestore(&mm->context.lock, flags); spin_unlock_irqrestore(&mm->context.lock, flags);
} }
void flush_dcache_page(struct page *page) void flush_dcache_folio(struct folio *folio)
{ {
unsigned long pfn = folio_pfn(folio);
struct address_space *mapping; struct address_space *mapping;
int this_cpu; int this_cpu;
@ -459,35 +473,35 @@ void flush_dcache_page(struct page *page)
* is merely the zero page. The 'bigcore' testcase in GDB * is merely the zero page. The 'bigcore' testcase in GDB
* causes this case to run millions of times. * causes this case to run millions of times.
*/ */
if (page == ZERO_PAGE(0)) if (is_zero_pfn(pfn))
return; return;
this_cpu = get_cpu(); this_cpu = get_cpu();
mapping = page_mapping_file(page); mapping = folio_flush_mapping(folio);
if (mapping && !mapping_mapped(mapping)) { if (mapping && !mapping_mapped(mapping)) {
int dirty = test_bit(PG_dcache_dirty, &page->flags); bool dirty = test_bit(PG_dcache_dirty, &folio->flags);
if (dirty) { if (dirty) {
int dirty_cpu = dcache_dirty_cpu(page); int dirty_cpu = dcache_dirty_cpu(folio);
if (dirty_cpu == this_cpu) if (dirty_cpu == this_cpu)
goto out; goto out;
smp_flush_dcache_page_impl(page, dirty_cpu); smp_flush_dcache_folio_impl(folio, dirty_cpu);
} }
set_dcache_dirty(page, this_cpu); set_dcache_dirty(folio, this_cpu);
} else { } else {
/* We could delay the flush for the !page_mapping /* We could delay the flush for the !page_mapping
* case too. But that case is for exec env/arg * case too. But that case is for exec env/arg
* pages and those are %99 certainly going to get * pages and those are %99 certainly going to get
* faulted into the tlb (and thus flushed) anyways. * faulted into the tlb (and thus flushed) anyways.
*/ */
flush_dcache_page_impl(page); flush_dcache_folio_impl(folio);
} }
out: out:
put_cpu(); put_cpu();
} }
EXPORT_SYMBOL(flush_dcache_page); EXPORT_SYMBOL(flush_dcache_folio);
void __kprobes flush_icache_range(unsigned long start, unsigned long end) void __kprobes flush_icache_range(unsigned long start, unsigned long end)
{ {
@ -2280,10 +2294,10 @@ void __init paging_init(void)
setup_page_offset(); setup_page_offset();
/* These build time checkes make sure that the dcache_dirty_cpu() /* These build time checkes make sure that the dcache_dirty_cpu()
* page->flags usage will work. * folio->flags usage will work.
* *
* When a page gets marked as dcache-dirty, we store the * When a page gets marked as dcache-dirty, we store the
* cpu number starting at bit 32 in the page->flags. Also, * cpu number starting at bit 32 in the folio->flags. Also,
* functions like clear_dcache_dirty_cpu use the cpu mask * functions like clear_dcache_dirty_cpu use the cpu mask
* in 13-bit signed-immediate instruction fields. * in 13-bit signed-immediate instruction fields.
*/ */

5
arch/sparc/mm/tlb.c
View file

@ -118,6 +118,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
unsigned long paddr, pfn = pte_pfn(orig); unsigned long paddr, pfn = pte_pfn(orig);
struct address_space *mapping; struct address_space *mapping;
struct page *page; struct page *page;
struct folio *folio;
if (!pfn_valid(pfn)) if (!pfn_valid(pfn))
goto no_cache_flush; goto no_cache_flush;
@ -127,13 +128,13 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
goto no_cache_flush; goto no_cache_flush;
/* A real file page? */ /* A real file page? */
mapping = page_mapping_file(page); mapping = folio_flush_mapping(folio);
if (!mapping) if (!mapping)
goto no_cache_flush; goto no_cache_flush;
paddr = (unsigned long) page_address(page); paddr = (unsigned long) page_address(page);
if ((paddr ^ vaddr) & (1 << 13)) if ((paddr ^ vaddr) & (1 << 13))
flush_dcache_page_all(mm, page); flush_dcache_folio_all(mm, folio);
} }
no_cache_flush: no_cache_flush: