Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945/* SPDX-License-Identifier: GPL-2.0 */#ifndef __LINUX_UACCESS_H__#define __LINUX_UACCESS_H__
#include <linux/cleanup.h>#include <linux/fault-inject-usercopy.h>#include <linux/instrumented.h>#include <linux/minmax.h>#include <linux/nospec.h>#include <linux/sched.h>#include <linux/ucopysize.h>
#include <asm/uaccess.h>
/* * Architectures that support memory tagging (assigning tags to memory regions, * embedding these tags into addresses that point to these memory regions, and * checking that the memory and the pointer tags match on memory accesses) * redefine this macro to strip tags from pointers. * * Passing down mm_struct allows to define untagging rules on per-process * basis. * * It's defined as noop for architectures that don't support memory tagging. */#ifndef untagged_addr#define untagged_addr(addr) (addr)#endif
#ifndef untagged_addr_remote#define untagged_addr_remote(mm, addr) ({ \ mmap_assert_locked(mm); \ untagged_addr(addr); \})#endif
#ifdef masked_user_access_begin #define can_do_masked_user_access() 1# ifndef masked_user_write_access_begin# define masked_user_write_access_begin masked_user_access_begin# endif# ifndef masked_user_read_access_begin# define masked_user_read_access_begin masked_user_access_begin#endif#else #define can_do_masked_user_access() 0 #define masked_user_access_begin(src) NULL #define masked_user_read_access_begin(src) NULL #define masked_user_write_access_begin(src) NULL #define mask_user_address(src) (src)#endif
/* * Architectures should provide two primitives (raw_copy_{to,from}_user()) * and get rid of their private instances of copy_{to,from}_user() and * __copy_{to,from}_user{,_inatomic}(). * * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and * return the amount left to copy. They should assume that access_ok() has * already been checked (and succeeded); they should *not* zero-pad anything. * No KASAN or object size checks either - those belong here. * * Both of these functions should attempt to copy size bytes starting at from * into the area starting at to. They must not fetch or store anything * outside of those areas. Return value must be between 0 (everything * copied successfully) and size (nothing copied). * * If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting * at to must become equal to the bytes fetched from the corresponding area * starting at from. All data past to + size - N must be left unmodified. * * If copying succeeds, the return value must be 0. If some data cannot be * fetched, it is permitted to copy less than had been fetched; the only * hard requirement is that not storing anything at all (i.e. returning size) * should happen only when nothing could be copied. In other words, you don't * have to squeeze as much as possible - it is allowed, but not necessary. * * For raw_copy_from_user() to always points to kernel memory and no faults * on store should happen. Interpretation of from is affected by set_fs(). * For raw_copy_to_user() it's the other way round. * * Both can be inlined - it's up to architectures whether it wants to bother * with that. They should not be used directly; they are used to implement * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) * that are used instead. Out of those, __... ones are inlined. Plain * copy_{to,from}_user() might or might not be inlined. If you want them * inlined, have asm/uaccess.h define INLINE_COPY_USER. * * NOTE: only copy_from_user() zero-pads the destination in case of short copy. * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything * at all; their callers absolutely must check the return value. * * Biarch ones should also provide raw_copy_in_user() - similar to the above, * but both source and destination are __user pointers (affected by set_fs() * as usual) and both source and destination can trigger faults. */
static __always_inline __must_check unsigned long__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n){ unsigned long res;
instrument_copy_from_user_before(to, from, n); check_object_size(to, n, false); res = raw_copy_from_user(to, from, n); instrument_copy_from_user_after(to, from, n, res); return res;}
static __always_inline __must_check unsigned long__copy_from_user(void *to, const void __user *from, unsigned long n){ unsigned long res;
might_fault(); instrument_copy_from_user_before(to, from, n); if (should_fail_usercopy()) return n; check_object_size(to, n, false); res = raw_copy_from_user(to, from, n); instrument_copy_from_user_after(to, from, n, res); return res;}
/** * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. * @to: Destination address, in user space. * @from: Source address, in kernel space. * @n: Number of bytes to copy. * * Context: User context only. * * Copy data from kernel space to user space. Caller must check * the specified block with access_ok() before calling this function. * The caller should also make sure he pins the user space address * so that we don't result in page fault and sleep. */static __always_inline __must_check unsigned long__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n){ if (should_fail_usercopy()) return n; instrument_copy_to_user(to, from, n); check_object_size(from, n, true); return raw_copy_to_user(to, from, n);}
static __always_inline __must_check unsigned long__copy_to_user(void __user *to, const void *from, unsigned long n){ might_fault(); if (should_fail_usercopy()) return n; instrument_copy_to_user(to, from, n); check_object_size(from, n, true); return raw_copy_to_user(to, from, n);}
/* * Architectures that #define INLINE_COPY_USER use this function * directly in the normal copy_to/from_user(), the other ones go * through an extern _copy_to/from_user(), which expands the same code * here. */static inline __must_check unsigned long_inline_copy_from_user(void *to, const void __user *from, unsigned long n){ unsigned long res = n; might_fault(); if (should_fail_usercopy()) goto fail; if (can_do_masked_user_access()) from = mask_user_address(from); else { if (!access_ok(from, n)) goto fail; /* * Ensure that bad access_ok() speculation will not * lead to nasty side effects *after* the copy is * finished: */ barrier_nospec(); } instrument_copy_from_user_before(to, from, n); res = raw_copy_from_user(to, from, n); instrument_copy_from_user_after(to, from, n, res); if (likely(!res)) return 0;fail: memset(to + (n - res), 0, res); return res;}
static inline __must_check unsigned long_inline_copy_to_user(void __user *to, const void *from, unsigned long n){ might_fault(); if (should_fail_usercopy()) return n; if (access_ok(to, n)) { instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } return n;}#ifdef INLINE_COPY_USER# define _copy_to_user _inline_copy_to_user# define _copy_from_user _inline_copy_from_user#elseextern __must_check unsigned long_copy_from_user(void *, const void __user *, unsigned long);
extern __must_check unsigned long_copy_to_user(void __user *, const void *, unsigned long);#endif
static __always_inline unsigned long __must_checkcopy_from_user(void *to, const void __user *from, unsigned long n){ if (!check_copy_size(to, n, false)) return n; return _copy_from_user(to, from, n);}
static __always_inline unsigned long __must_checkcopy_to_user(void __user *to, const void *from, unsigned long n){ if (!check_copy_size(from, n, true)) return n; return _copy_to_user(to, from, n);}
#ifndef copy_mc_to_kernel/* * Without arch opt-in this generic copy_mc_to_kernel() will not handle * #MC (or arch equivalent) during source read. */static inline unsigned long __must_checkcopy_mc_to_kernel(void *dst, const void *src, size_t cnt){ memcpy(dst, src, cnt); return 0;}#endif
static __always_inline void pagefault_disabled_inc(void){ current->pagefault_disabled++;}
static __always_inline void pagefault_disabled_dec(void){ current->pagefault_disabled--;}
/* * These routines enable/disable the pagefault handler. If disabled, it will * not take any locks and go straight to the fixup table. * * User access methods will not sleep when called from a pagefault_disabled() * environment. */static inline void pagefault_disable(void){ pagefault_disabled_inc(); /* * make sure to have issued the store before a pagefault * can hit. */ barrier();}
static inline void pagefault_enable(void){ /* * make sure to issue those last loads/stores before enabling * the pagefault handler again. */ barrier(); pagefault_disabled_dec();}
/* * Is the pagefault handler disabled? If so, user access methods will not sleep. */static inline bool pagefault_disabled(void){ return current->pagefault_disabled != 0;}
/* * The pagefault handler is in general disabled by pagefault_disable() or * when in irq context (via in_atomic()). * * This function should only be used by the fault handlers. Other users should * stick to pagefault_disabled(). * Please NEVER use preempt_disable() to disable the fault handler. With * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. */#define faulthandler_disabled() (pagefault_disabled() || in_atomic())
DEFINE_LOCK_GUARD_0(pagefault, pagefault_disable(), pagefault_enable())
#ifndef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
/** * probe_subpage_writeable: probe the user range for write faults at sub-page * granularity (e.g. arm64 MTE) * @uaddr: start of address range * @size: size of address range * * Returns 0 on success, the number of bytes not probed on fault. * * It is expected that the caller checked for the write permission of each * page in the range either by put_user() or GUP. The architecture port can * implement a more efficient get_user() probing if the same sub-page faults * are triggered by either a read or a write. */static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size){ return 0;}
#endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
#ifndef ARCH_HAS_NONTEMPORAL_UACCESS
static inline __must_check unsigned longcopy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n){ if (can_do_masked_user_access()) from = mask_user_address(from); else if (!access_ok(from, n)) return n; return __copy_from_user_inatomic(to, from, n);}
#endif /* ARCH_HAS_NONTEMPORAL_UACCESS */
extern __must_check int check_zeroed_user(const void __user *from, size_t size);
/** * copy_struct_from_user: copy a struct from userspace * @dst: Destination address, in kernel space. This buffer must be @ksize * bytes long. * @ksize: Size of @dst struct. * @src: Source address, in userspace. * @usize: (Alleged) size of @src struct. * * Copies a struct from userspace to kernel space, in a way that guarantees * backwards-compatibility for struct syscall arguments (as long as future * struct extensions are made such that all new fields are *appended* to the * old struct, and zeroed-out new fields have the same meaning as the old * struct). * * @ksize is just sizeof(*dst), and @usize should've been passed by userspace. * The recommended usage is something like the following: * * SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize) * { * int err; * struct foo karg = {}; * * if (usize > PAGE_SIZE) * return -E2BIG; * if (usize < FOO_SIZE_VER0) * return -EINVAL; * * err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); * if (err) * return err; * * // ... * } * * There are three cases to consider: * * If @usize == @ksize, then it's copied verbatim. * * If @usize < @ksize, then the userspace has passed an old struct to a * newer kernel. The rest of the trailing bytes in @dst (@ksize - @usize) * are to be zero-filled. * * If @usize > @ksize, then the userspace has passed a new struct to an * older kernel. The trailing bytes unknown to the kernel (@usize - @ksize) * are checked to ensure they are zeroed, otherwise -E2BIG is returned. * * Returns (in all cases, some data may have been copied): * * -E2BIG: (@usize > @ksize) and there are non-zero trailing bytes in @src. * * -EFAULT: access to userspace failed. */static __always_inline __must_check intcopy_struct_from_user(void *dst, size_t ksize, const void __user *src, size_t usize){ size_t size = min(ksize, usize); size_t rest = max(ksize, usize) - size;
/* Double check if ksize is larger than a known object size. */ if (WARN_ON_ONCE(ksize > __builtin_object_size(dst, 1))) return -E2BIG;
/* Deal with trailing bytes. */ if (usize < ksize) { memset(dst + size, 0, rest); } else if (usize > ksize) { int ret = check_zeroed_user(src + size, rest); if (ret <= 0) return ret ?: -E2BIG; } /* Copy the interoperable parts of the struct. */ if (copy_from_user(dst, src, size)) return -EFAULT; return 0;}
/** * copy_struct_to_user: copy a struct to userspace * @dst: Destination address, in userspace. This buffer must be @ksize * bytes long. * @usize: (Alleged) size of @dst struct. * @src: Source address, in kernel space. * @ksize: Size of @src struct. * @ignored_trailing: Set to %true if there was a non-zero byte in @src that * userspace cannot see because they are using an smaller struct. * * Copies a struct from kernel space to userspace, in a way that guarantees * backwards-compatibility for struct syscall arguments (as long as future * struct extensions are made such that all new fields are *appended* to the * old struct, and zeroed-out new fields have the same meaning as the old * struct). * * Some syscalls may wish to make sure that userspace knows about everything in * the struct, and if there is a non-zero value that userspce doesn't know * about, they want to return an error (such as -EMSGSIZE) or have some other * fallback (such as adding a "you're missing some information" flag). If * @ignored_trailing is non-%NULL, it will be set to %true if there was a * non-zero byte that could not be copied to userspace (ie. was past @usize). * * While unconditionally returning an error in this case is the simplest * solution, for maximum backward compatibility you should try to only return * -EMSGSIZE if the user explicitly requested the data that couldn't be copied. * Note that structure sizes can change due to header changes and simple * recompilations without code changes(!), so if you care about * @ignored_trailing you probably want to make sure that any new field data is * associated with a flag. Otherwise you might assume that a program knows * about data it does not. * * @ksize is just sizeof(*src), and @usize should've been passed by userspace. * The recommended usage is something like the following: * * SYSCALL_DEFINE2(foobar, struct foo __user *, uarg, size_t, usize) * { * int err; * bool ignored_trailing; * struct foo karg = {}; * * if (usize > PAGE_SIZE) * return -E2BIG; * if (usize < FOO_SIZE_VER0) * return -EINVAL; * * // ... modify karg somehow ... * * err = copy_struct_to_user(uarg, usize, &karg, sizeof(karg), * &ignored_trailing); * if (err) * return err; * if (ignored_trailing) * return -EMSGSIZE: * * // ... * } * * There are three cases to consider: * * If @usize == @ksize, then it's copied verbatim. * * If @usize < @ksize, then the kernel is trying to pass userspace a newer * struct than it supports. Thus we only copy the interoperable portions * (@usize) and ignore the rest (but @ignored_trailing is set to %true if * any of the trailing (@ksize - @usize) bytes are non-zero). * * If @usize > @ksize, then the kernel is trying to pass userspace an older * struct than userspace supports. In order to make sure the * unknown-to-the-kernel fields don't contain garbage values, we zero the * trailing (@usize - @ksize) bytes. * * Returns (in all cases, some data may have been copied): * * -EFAULT: access to userspace failed. */static __always_inline __must_check intcopy_struct_to_user(void __user *dst, size_t usize, const void *src, size_t ksize, bool *ignored_trailing){ size_t size = min(ksize, usize); size_t rest = max(ksize, usize) - size;
/* Double check if ksize is larger than a known object size. */ if (WARN_ON_ONCE(ksize > __builtin_object_size(src, 1))) return -E2BIG;
/* Deal with trailing bytes. */ if (usize > ksize) { if (clear_user(dst + size, rest)) return -EFAULT; } if (ignored_trailing) *ignored_trailing = usize < ksize && memchr_inv(src + size, 0, rest) != NULL; /* Copy the interoperable parts of the struct. */ if (copy_to_user(dst, src, size)) return -EFAULT; return 0;}
static __always_inline void__copy_struct_generic_bounce_buffer(void *dst, size_t dstsize, const void *src, size_t srcsize, bool *ignored_trailing){ size_t size = min(dstsize, srcsize); size_t rest = max(dstsize, srcsize) - size;
/* Deal with trailing bytes. */ if (dstsize > srcsize) memset(dst + size, 0, rest); if (ignored_trailing) *ignored_trailing = dstsize < srcsize && memchr_inv(src + size, 0, rest) != NULL; /* Copy the interoperable parts of the struct. */ memcpy(dst, src, size);}
/** * This is like copy_struct_from_user(), but the * src buffer was already copied into a kernel * bounce buffer, so it will never return -EFAULT. */static __always_inline __must_check intcopy_struct_from_bounce_buffer(void *dst, size_t dstsize, const void *src, size_t srcsize){ bool ignored_trailing;
/* Double check if ksize is larger than a known object size. */ if (WARN_ON_ONCE(dstsize > __builtin_object_size(dst, 1))) return -E2BIG;
__copy_struct_generic_bounce_buffer(dst, dstsize, src, srcsize, &ignored_trailing); if (unlikely(ignored_trailing)) return -E2BIG;
return 0;}
/** * This is like copy_struct_to_user(), but the * dst buffer is a kernel bounce buffer instead * of a direct userspace buffer, so it will never return -EFAULT. */static __always_inline __must_check intcopy_struct_to_bounce_buffer(void *dst, size_t dstsize, const void *src, size_t srcsize, bool *ignored_trailing){ /* Double check if srcsize is larger than a known object size. */ if (WARN_ON_ONCE(srcsize > __builtin_object_size(src, 1))) return -E2BIG;
__copy_struct_generic_bounce_buffer(dst, dstsize, src, srcsize, ignored_trailing); return 0;}
bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size);
long copy_from_kernel_nofault(void *dst, const void *src, size_t size);long notrace copy_to_kernel_nofault(void *dst, const void *src, size_t size);
long copy_from_user_nofault(void *dst, const void __user *src, size_t size);long notrace copy_to_user_nofault(void __user *dst, const void *src, size_t size);
long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count);
long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, long count);long strnlen_user_nofault(const void __user *unsafe_addr, long count);
#ifdef arch_get_kernel_nofault/* * Wrap the architecture implementation so that @label can be outside of a * cleanup() scope. A regular C goto works correctly, but ASM goto does * not. Clang rejects such an attempt, but GCC silently emits buggy code. */#define __get_kernel_nofault(dst, src, type, label) \do { \ __label__ local_label; \ arch_get_kernel_nofault(dst, src, type, local_label); \ if (0) { \ local_label: \ goto label; \ } \} while (0)
#define __put_kernel_nofault(dst, src, type, label) \do { \ __label__ local_label; \ arch_put_kernel_nofault(dst, src, type, local_label); \ if (0) { \ local_label: \ goto label; \ } \} while (0)
#elif !defined(__get_kernel_nofault) /* arch_get_kernel_nofault */
#define __get_kernel_nofault(dst, src, type, label) \do { \ type __user *p = (type __force __user *)(src); \ type data; \ if (__get_user(data, p)) \ goto label; \ *(type *)dst = data; \} while (0)
#define __put_kernel_nofault(dst, src, type, label) \do { \ type __user *p = (type __force __user *)(dst); \ type data = *(type *)src; \ if (__put_user(data, p)) \ goto label; \} while (0)
#endif /* !__get_kernel_nofault */
/** * get_kernel_nofault(): safely attempt to read from a location * @val: read into this variable * @ptr: address to read from * * Returns 0 on success, or -EFAULT. */#define get_kernel_nofault(val, ptr) ({ \ const typeof(val) *__gk_ptr = (ptr); \ copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\})
#ifdef user_access_begin
#ifdef arch_unsafe_get_user/* * Wrap the architecture implementation so that @label can be outside of a * cleanup() scope. A regular C goto works correctly, but ASM goto does * not. Clang rejects such an attempt, but GCC silently emits buggy code. * * Some architectures use internal local labels already, but this extra * indirection here is harmless because the compiler optimizes it out * completely in any case. This construct just ensures that the ASM GOTO * target is always in the local scope. The C goto 'label' works correctly * when leaving a cleanup() scope. */#define unsafe_get_user(x, ptr, label) \do { \ __label__ local_label; \ arch_unsafe_get_user(x, ptr, local_label); \ if (0) { \ local_label: \ goto label; \ } \} while (0)
#define unsafe_put_user(x, ptr, label) \do { \ __label__ local_label; \ arch_unsafe_put_user(x, ptr, local_label); \ if (0) { \ local_label: \ goto label; \ } \} while (0)#endif /* arch_unsafe_get_user */
#else /* user_access_begin */#define user_access_begin(ptr,len) access_ok(ptr, len)#define user_access_end() do { } while (0)#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)#define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)static inline unsigned long user_access_save(void) { return 0UL; }static inline void user_access_restore(unsigned long flags) { }#endif /* !user_access_begin */
#ifndef user_write_access_begin#define user_write_access_begin user_access_begin#define user_write_access_end user_access_end#endif#ifndef user_read_access_begin#define user_read_access_begin user_access_begin#define user_read_access_end user_access_end#endif
#ifndef unsafe_atomic_store_release_user# define unsafe_atomic_store_release_user(val, uptr, elbl) \ do { \ if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) \ smp_mb(); \ else \ barrier(); \ unsafe_put_user(val, uptr, elbl); \ } while (0)#endif
/* Define RW variant so the below _mode macro expansion works */#define masked_user_rw_access_begin(u) masked_user_access_begin(u)#define user_rw_access_begin(u, s) user_access_begin(u, s)
/* Scoped user access */
/* Cleanup wrapper functions */static __always_inline void __scoped_user_read_access_end(const void *p){ user_read_access_end();};static __always_inline void __scoped_user_write_access_end(const void *p){ user_write_access_end();};static __always_inline void __scoped_user_rw_access_end(const void *p){ user_access_end();};
/** * __scoped_user_access_begin - Start a scoped user access * @mode: The mode of the access class (read, write, rw) * @uptr: The pointer to access user space memory * @size: Size of the access * @elbl: Error label to goto when the access region is rejected * * Internal helper for __scoped_user_access(). Don't use directly. */#define __scoped_user_access_begin(mode, uptr, size, elbl) \({ \ typeof(uptr) __retptr; \ \ if (can_do_masked_user_access()) { \ __retptr = masked_user_##mode##_access_begin(uptr); \ } else { \ __retptr = uptr; \ if (!user_##mode##_access_begin(uptr, size)) \ goto elbl; \ } \ __retptr; \})
/** * __scoped_user_access - Open a scope for user access * @mode: The mode of the access class (read, write, rw) * @uptr: The pointer to access user space memory * @size: Size of the access * @elbl: Error label to goto when the access region is rejected. It * must be placed outside the scope * * If the user access function inside the scope requires a fault label, it * can use @elbl or a different label outside the scope, which requires * that user access which is implemented with ASM GOTO has been properly * wrapped. See unsafe_get_user() for reference. * * scoped_user_rw_access(ptr, efault) { * unsafe_get_user(rval, &ptr->rval, efault); * unsafe_put_user(wval, &ptr->wval, efault); * } * return 0; * efault: * return -EFAULT; * * The scope is internally implemented as a autoterminating nested for() * loop, which can be left with 'return', 'break' and 'goto' at any * point. * * When the scope is left user_##@_mode##_access_end() is automatically * invoked. * * When the architecture supports masked user access and the access region * which is determined by @uptr and @size is not a valid user space * address, i.e. < TASK_SIZE, the scope sets the pointer to a faulting user * space address and does not terminate early. This optimizes for the good * case and lets the performance uncritical bad case go through the fault. * * The eventual modification of the pointer is limited to the scope. * Outside of the scope the original pointer value is unmodified, so that * the original pointer value is available for diagnostic purposes in an * out of scope fault path. * * Nesting scoped user access into a user access scope is invalid and fails * the build. Nesting into other guards, e.g. pagefault is safe. * * The masked variant does not check the size of the access and relies on a * mapping hole (e.g. guard page) to catch an out of range pointer, the * first access to user memory inside the scope has to be within * @uptr ... @uptr + PAGE_SIZE - 1 * * Don't use directly. Use scoped_masked_user_$MODE_access() instead. */#define __scoped_user_access(mode, uptr, size, elbl) \for (bool done = false; !done; done = true) \ for (auto _tmpptr = __scoped_user_access_begin(mode, uptr, size, elbl); \ !done; done = true) \ /* Force modified pointer usage within the scope */ \ for (const auto uptr __cleanup(__scoped_user_##mode##_access_end) = \ _tmpptr; !done; done = true)
/** * scoped_user_read_access_size - Start a scoped user read access with given size * @usrc: Pointer to the user space address to read from * @size: Size of the access starting from @usrc * @elbl: Error label to goto when the access region is rejected * * For further information see __scoped_user_access() above. */#define scoped_user_read_access_size(usrc, size, elbl) \ __scoped_user_access(read, usrc, size, elbl)
/** * scoped_user_read_access - Start a scoped user read access * @usrc: Pointer to the user space address to read from * @elbl: Error label to goto when the access region is rejected * * The size of the access starting from @usrc is determined via sizeof(*@usrc)). * * For further information see __scoped_user_access() above. */#define scoped_user_read_access(usrc, elbl) \ scoped_user_read_access_size(usrc, sizeof(*(usrc)), elbl)
/** * scoped_user_write_access_size - Start a scoped user write access with given size * @udst: Pointer to the user space address to write to * @size: Size of the access starting from @udst * @elbl: Error label to goto when the access region is rejected * * For further information see __scoped_user_access() above. */#define scoped_user_write_access_size(udst, size, elbl) \ __scoped_user_access(write, udst, size, elbl)
/** * scoped_user_write_access - Start a scoped user write access * @udst: Pointer to the user space address to write to * @elbl: Error label to goto when the access region is rejected * * The size of the access starting from @udst is determined via sizeof(*@udst)). * * For further information see __scoped_user_access() above. */#define scoped_user_write_access(udst, elbl) \ scoped_user_write_access_size(udst, sizeof(*(udst)), elbl)
/** * scoped_user_rw_access_size - Start a scoped user read/write access with given size * @uptr: Pointer to the user space address to read from and write to * @size: Size of the access starting from @uptr * @elbl: Error label to goto when the access region is rejected * * For further information see __scoped_user_access() above. */#define scoped_user_rw_access_size(uptr, size, elbl) \ __scoped_user_access(rw, uptr, size, elbl)
/** * scoped_user_rw_access - Start a scoped user read/write access * @uptr: Pointer to the user space address to read from and write to * @elbl: Error label to goto when the access region is rejected * * The size of the access starting from @uptr is determined via sizeof(*@uptr)). * * For further information see __scoped_user_access() above. */#define scoped_user_rw_access(uptr, elbl) \ scoped_user_rw_access_size(uptr, sizeof(*(uptr)), elbl)
/** * get_user_inline - Read user data inlined * @val: The variable to store the value read from user memory * @usrc: Pointer to the user space memory to read from * * Return: 0 if successful, -EFAULT when faulted * * Inlined variant of get_user(). Only use when there is a demonstrable * performance reason. */#define get_user_inline(val, usrc) \({ \ __label__ efault; \ typeof(usrc) _tmpsrc = usrc; \ int _ret = 0; \ \ scoped_user_read_access(_tmpsrc, efault) \ unsafe_get_user(val, _tmpsrc, efault); \ if (0) { \ efault: \ _ret = -EFAULT; \ } \ _ret; \})
/** * put_user_inline - Write to user memory inlined * @val: The value to write * @udst: Pointer to the user space memory to write to * * Return: 0 if successful, -EFAULT when faulted * * Inlined variant of put_user(). Only use when there is a demonstrable * performance reason. */#define put_user_inline(val, udst) \({ \ __label__ efault; \ typeof(udst) _tmpdst = udst; \ int _ret = 0; \ \ scoped_user_write_access(_tmpdst, efault) \ unsafe_put_user(val, _tmpdst, efault); \ if (0) { \ efault: \ _ret = -EFAULT; \ } \ _ret; \})
#ifdef CONFIG_HARDENED_USERCOPYvoid __noreturn usercopy_abort(const char *name, const char *detail, bool to_user, unsigned long offset, unsigned long len);#endif
#endif /* __LINUX_UACCESS_H__ */