mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

x86/mm: Reduce untagged_addr() overhead for systems without LAM

Browse source 
Use alternatives to reduce untagged_addr() overhead.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lore.kernel.org/all/20230312112612.31869-8-kirill.shutemov%40linux.intel.com
This commit is contained in:
Kirill A. Shutemov 2023-03-12 14:26:02 +03:00 committed by Dave Hansen
parent 74c228d20a
commit e0bddc19ba
2 changed files with 37 additions and 12 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

8
arch/x86/include/asm/disabled-features.h
View file

@ -75,6 +75,12 @@
# define DISABLE_CALL_DEPTH_TRACKING (1 << (X86_FEATURE_CALL_DEPTH & 31))
#endif
#ifdef CONFIG_ADDRESS_MASKING
# define DISABLE_LAM 0
#else
# define DISABLE_LAM (1 << (X86_FEATURE_LAM & 31))
#endif
#ifdef CONFIG_INTEL_IOMMU_SVM
# define DISABLE_ENQCMD 0
#else
@ -115,7 +121,7 @@
#define DISABLED_MASK10 0
#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
DISABLE_CALL_DEPTH_TRACKING)
#define DISABLED_MASK12 0
#define DISABLED_MASK12 (DISABLE_LAM)
#define DISABLED_MASK13 0
#define DISABLED_MASK14 0
#define DISABLED_MASK15 0

41
arch/x86/include/asm/uaccess.h
View file

@ -9,6 +9,7 @@
#include <linux/kasan-checks.h>
#include <linux/mm_types.h>
#include <linux/string.h>
#include <linux/mmap_lock.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/smap.h>
@ -30,26 +31,44 @@ static inline bool pagefault_disabled(void);
* Magic with the 'sign' allows to untag userspace pointer without any branches
* while leaving kernel addresses intact.
*/
static inline unsigned long __untagged_addr(unsigned long addr,
unsigned long mask)
static inline unsigned long __untagged_addr(unsigned long addr)
{
long sign = addr >> 63;
long sign;
/*
* Refer tlbstate_untag_mask directly to avoid RIP-relative relocation
* in alternative instructions. The relocation gets wrong when gets
* copied to the target place.
*/
asm (ALTERNATIVE("",
"sar $63, %[sign]\n\t" /* user_ptr ? 0 : -1UL */
"or %%gs:tlbstate_untag_mask, %[sign]\n\t"
"and %[sign], %[addr]\n\t", X86_FEATURE_LAM)
: [addr] "+r" (addr), [sign] "=r" (sign)
: "m" (tlbstate_untag_mask), "[sign]" (addr));
addr &= mask | sign;
return addr;
}
#define untagged_addr(addr) ({ \
u64 __addr = (__force u64)(addr); \
__addr = __untagged_addr(__addr, current_untag_mask()); \
(__force __typeof__(addr))__addr; \
unsigned long __addr = (__force unsigned long)(addr); \
(__force __typeof__(addr))__untagged_addr(__addr); \
})
static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
unsigned long addr)
{
long sign = addr >> 63;
mmap_assert_locked(mm);
addr &= (mm)->context.untag_mask | sign;
return addr;
}
#define untagged_addr_remote(mm, addr) ({ \
u64 __addr = (__force u64)(addr); \
mmap_assert_locked(mm); \
__addr = __untagged_addr(__addr, (mm)->context.untag_mask); \
(__force __typeof__(addr))__addr; \
unsigned long __addr = (__force unsigned long)(addr); \
(__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
})
#else