linux/lib/strnlen_user.c
Linus Torvalds 2865baf540 x86: support user address masking instead of non-speculative conditional
The Spectre-v1 mitigations made "access_ok()" much more expensive, since
it has to serialize execution with the test for a valid user address.

All the normal user copy routines avoid this by just masking the user
address with a data-dependent mask instead, but the fast
"unsafe_user_read()" kind of patterms that were supposed to be a fast
case got slowed down.

This introduces a notion of using

	src = masked_user_access_begin(src);

to do the user address sanity using a data-dependent mask instead of the
more traditional conditional

	if (user_read_access_begin(src, len)) {

model.

This model only works for dense accesses that start at 'src' and on
architectures that have a guard region that is guaranteed to fault in
between the user space and the kernel space area.

With this, the user access doesn't need to be manually checked, because
a bad address is guaranteed to fault (by some architecture masking
trick: on x86-64 this involves just turning an invalid user address into
all ones, since we don't map the top of address space).

This only converts a couple of examples for now.  Example x86-64 code
generation for loading two words from user space:

        stac
        mov    %rax,%rcx
        sar    $0x3f,%rcx
        or     %rax,%rcx
        mov    (%rcx),%r13
        mov    0x8(%rcx),%r14
        clac

where all the error handling and -EFAULT is now purely handled out of
line by the exception path.

Of course, if the micro-architecture does badly at 'clac' and 'stac',
the above is still pitifully slow.  But at least we did as well as we
could.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2024-08-19 11:31:18 -07:00

130 lines
3.5 KiB
C

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/uaccess.h>
#include <linux/mm.h>
#include <linux/bitops.h>
#include <asm/word-at-a-time.h>
/*
* Do a strnlen, return length of string *with* final '\0'.
* 'count' is the user-supplied count, while 'max' is the
* address space maximum.
*
* Return 0 for exceptions (which includes hitting the address
* space maximum), or 'count+1' if hitting the user-supplied
* maximum count.
*
* NOTE! We can sometimes overshoot the user-supplied maximum
* if it fits in a aligned 'long'. The caller needs to check
* the return value against "> max".
*/
static __always_inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
{
const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
unsigned long align, res = 0;
unsigned long c;
/*
* Do everything aligned. But that means that we
* need to also expand the maximum..
*/
align = (sizeof(unsigned long) - 1) & (unsigned long)src;
src -= align;
max += align;
unsafe_get_user(c, (unsigned long __user *)src, efault);
c |= aligned_byte_mask(align);
for (;;) {
unsigned long data;
if (has_zero(c, &data, &constants)) {
data = prep_zero_mask(c, data, &constants);
data = create_zero_mask(data);
return res + find_zero(data) + 1 - align;
}
res += sizeof(unsigned long);
/* We already handled 'unsigned long' bytes. Did we do it all ? */
if (unlikely(max <= sizeof(unsigned long)))
break;
max -= sizeof(unsigned long);
unsafe_get_user(c, (unsigned long __user *)(src+res), efault);
}
res -= align;
/*
* Uhhuh. We hit 'max'. But was that the user-specified maximum
* too? If so, return the marker for "too long".
*/
if (res >= count)
return count+1;
/*
* Nope: we hit the address space limit, and we still had more
* characters the caller would have wanted. That's 0.
*/
efault:
return 0;
}
/**
* strnlen_user: - Get the size of a user string INCLUDING final NUL.
* @str: The string to measure.
* @count: Maximum count (including NUL character)
*
* Context: User context only. This function may sleep if pagefaults are
* enabled.
*
* Get the size of a NUL-terminated string in user space.
*
* Returns the size of the string INCLUDING the terminating NUL.
* If the string is too long, returns a number larger than @count. User
* has to check the return value against "> count".
* On exception (or invalid count), returns 0.
*
* NOTE! You should basically never use this function. There is
* almost never any valid case for using the length of a user space
* string, since the string can be changed at any time by other
* threads. Use "strncpy_from_user()" instead to get a stable copy
* of the string.
*/
long strnlen_user(const char __user *str, long count)
{
unsigned long max_addr, src_addr;
if (unlikely(count <= 0))
return 0;
if (can_do_masked_user_access()) {
long retval;
str = masked_user_access_begin(str);
retval = do_strnlen_user(str, count, count);
user_read_access_end();
return retval;
}
max_addr = TASK_SIZE_MAX;
src_addr = (unsigned long)untagged_addr(str);
if (likely(src_addr < max_addr)) {
unsigned long max = max_addr - src_addr;
long retval;
/*
* Truncate 'max' to the user-specified limit, so that
* we only have one limit we need to check in the loop
*/
if (max > count)
max = count;
if (user_read_access_begin(str, max)) {
retval = do_strnlen_user(str, count, max);
user_read_access_end();
return retval;
}
}
return 0;
}
EXPORT_SYMBOL(strnlen_user);