forked from mirrors/linux
		
	 2865baf540
			
		
	
	
		2865baf540
		
	
	
	
	
		
			
			The Spectre-v1 mitigations made "access_ok()" much more expensive, since
it has to serialize execution with the test for a valid user address.
All the normal user copy routines avoid this by just masking the user
address with a data-dependent mask instead, but the fast
"unsafe_user_read()" kind of patterms that were supposed to be a fast
case got slowed down.
This introduces a notion of using
	src = masked_user_access_begin(src);
to do the user address sanity using a data-dependent mask instead of the
more traditional conditional
	if (user_read_access_begin(src, len)) {
model.
This model only works for dense accesses that start at 'src' and on
architectures that have a guard region that is guaranteed to fault in
between the user space and the kernel space area.
With this, the user access doesn't need to be manually checked, because
a bad address is guaranteed to fault (by some architecture masking
trick: on x86-64 this involves just turning an invalid user address into
all ones, since we don't map the top of address space).
This only converts a couple of examples for now.  Example x86-64 code
generation for loading two words from user space:
        stac
        mov    %rax,%rcx
        sar    $0x3f,%rcx
        or     %rax,%rcx
        mov    (%rcx),%r13
        mov    0x8(%rcx),%r14
        clac
where all the error handling and -EFAULT is now purely handled out of
line by the exception path.
Of course, if the micro-architecture does badly at 'clac' and 'stac',
the above is still pitifully slow.  But at least we did as well as we
could.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
		
	
			
		
			
				
	
	
		
			129 lines
		
	
	
	
		
			3.5 KiB
		
	
	
	
		
			C
		
	
	
	
	
	
			
		
		
	
	
			129 lines
		
	
	
	
		
			3.5 KiB
		
	
	
	
		
			C
		
	
	
	
	
	
| // SPDX-License-Identifier: GPL-2.0
 | |
| #include <linux/kernel.h>
 | |
| #include <linux/export.h>
 | |
| #include <linux/uaccess.h>
 | |
| #include <linux/mm.h>
 | |
| #include <linux/bitops.h>
 | |
| 
 | |
| #include <asm/word-at-a-time.h>
 | |
| 
 | |
| /*
 | |
|  * Do a strnlen, return length of string *with* final '\0'.
 | |
|  * 'count' is the user-supplied count, while 'max' is the
 | |
|  * address space maximum.
 | |
|  *
 | |
|  * Return 0 for exceptions (which includes hitting the address
 | |
|  * space maximum), or 'count+1' if hitting the user-supplied
 | |
|  * maximum count.
 | |
|  *
 | |
|  * NOTE! We can sometimes overshoot the user-supplied maximum
 | |
|  * if it fits in a aligned 'long'. The caller needs to check
 | |
|  * the return value against "> max".
 | |
|  */
 | |
| static __always_inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
 | |
| {
 | |
| 	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
 | |
| 	unsigned long align, res = 0;
 | |
| 	unsigned long c;
 | |
| 
 | |
| 	/*
 | |
| 	 * Do everything aligned. But that means that we
 | |
| 	 * need to also expand the maximum..
 | |
| 	 */
 | |
| 	align = (sizeof(unsigned long) - 1) & (unsigned long)src;
 | |
| 	src -= align;
 | |
| 	max += align;
 | |
| 
 | |
| 	unsafe_get_user(c, (unsigned long __user *)src, efault);
 | |
| 	c |= aligned_byte_mask(align);
 | |
| 
 | |
| 	for (;;) {
 | |
| 		unsigned long data;
 | |
| 		if (has_zero(c, &data, &constants)) {
 | |
| 			data = prep_zero_mask(c, data, &constants);
 | |
| 			data = create_zero_mask(data);
 | |
| 			return res + find_zero(data) + 1 - align;
 | |
| 		}
 | |
| 		res += sizeof(unsigned long);
 | |
| 		/* We already handled 'unsigned long' bytes. Did we do it all ? */
 | |
| 		if (unlikely(max <= sizeof(unsigned long)))
 | |
| 			break;
 | |
| 		max -= sizeof(unsigned long);
 | |
| 		unsafe_get_user(c, (unsigned long __user *)(src+res), efault);
 | |
| 	}
 | |
| 	res -= align;
 | |
| 
 | |
| 	/*
 | |
| 	 * Uhhuh. We hit 'max'. But was that the user-specified maximum
 | |
| 	 * too? If so, return the marker for "too long".
 | |
| 	 */
 | |
| 	if (res >= count)
 | |
| 		return count+1;
 | |
| 
 | |
| 	/*
 | |
| 	 * Nope: we hit the address space limit, and we still had more
 | |
| 	 * characters the caller would have wanted. That's 0.
 | |
| 	 */
 | |
| efault:
 | |
| 	return 0;
 | |
| }
 | |
| 
 | |
| /**
 | |
|  * strnlen_user: - Get the size of a user string INCLUDING final NUL.
 | |
|  * @str: The string to measure.
 | |
|  * @count: Maximum count (including NUL character)
 | |
|  *
 | |
|  * Context: User context only. This function may sleep if pagefaults are
 | |
|  *          enabled.
 | |
|  *
 | |
|  * Get the size of a NUL-terminated string in user space.
 | |
|  *
 | |
|  * Returns the size of the string INCLUDING the terminating NUL.
 | |
|  * If the string is too long, returns a number larger than @count. User
 | |
|  * has to check the return value against "> count".
 | |
|  * On exception (or invalid count), returns 0.
 | |
|  *
 | |
|  * NOTE! You should basically never use this function. There is
 | |
|  * almost never any valid case for using the length of a user space
 | |
|  * string, since the string can be changed at any time by other
 | |
|  * threads. Use "strncpy_from_user()" instead to get a stable copy
 | |
|  * of the string.
 | |
|  */
 | |
| long strnlen_user(const char __user *str, long count)
 | |
| {
 | |
| 	unsigned long max_addr, src_addr;
 | |
| 
 | |
| 	if (unlikely(count <= 0))
 | |
| 		return 0;
 | |
| 
 | |
| 	if (can_do_masked_user_access()) {
 | |
| 		long retval;
 | |
| 
 | |
| 		str = masked_user_access_begin(str);
 | |
| 		retval = do_strnlen_user(str, count, count);
 | |
| 		user_read_access_end();
 | |
| 		return retval;
 | |
| 	}
 | |
| 
 | |
| 	max_addr = TASK_SIZE_MAX;
 | |
| 	src_addr = (unsigned long)untagged_addr(str);
 | |
| 	if (likely(src_addr < max_addr)) {
 | |
| 		unsigned long max = max_addr - src_addr;
 | |
| 		long retval;
 | |
| 
 | |
| 		/*
 | |
| 		 * Truncate 'max' to the user-specified limit, so that
 | |
| 		 * we only have one limit we need to check in the loop
 | |
| 		 */
 | |
| 		if (max > count)
 | |
| 			max = count;
 | |
| 
 | |
| 		if (user_read_access_begin(str, max)) {
 | |
| 			retval = do_strnlen_user(str, count, max);
 | |
| 			user_read_access_end();
 | |
| 			return retval;
 | |
| 		}
 | |
| 	}
 | |
| 	return 0;
 | |
| }
 | |
| EXPORT_SYMBOL(strnlen_user);
 |