Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679// SPDX-License-Identifier: GPL-2.0
//! Slices to user space memory regions.//!//! C header: [`include/linux/uaccess.h`](srctree/include/linux/uaccess.h)
use crate::{ alloc::{Allocator, Flags}, bindings, dma::Coherent, error::Result, ffi::{c_char, c_void}, fs::file, prelude::*, ptr::KnownSize, transmute::{AsBytes, FromBytes},};use core::mem::{size_of, MaybeUninit};
/// A pointer into userspace.////// This is the Rust equivalent to C pointers tagged with `__user`.#[repr(transparent)]#[derive(Copy, Clone, Zeroable)]pub struct UserPtr(*mut c_void);
impl UserPtr { /// Create a `UserPtr` from an integer representing the userspace address. #[inline] pub fn from_addr(addr: usize) -> Self { Self(addr as *mut c_void) }
/// Create a `UserPtr` from a pointer representing the userspace address. #[inline] pub fn from_ptr(addr: *mut c_void) -> Self { Self(addr) }
/// Cast this userspace pointer to a raw const void pointer. /// /// It is up to the caller to use the returned pointer correctly. #[inline] pub fn as_const_ptr(self) -> *const c_void { self.0 }
/// Cast this userspace pointer to a raw mutable void pointer. /// /// It is up to the caller to use the returned pointer correctly. #[inline] pub fn as_mut_ptr(self) -> *mut c_void { self.0 }
/// Increment this user pointer by `add` bytes. /// /// This addition is wrapping, so wrapping around the address space does not result in a panic /// even if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled. #[inline] pub fn wrapping_byte_add(self, add: usize) -> UserPtr { UserPtr(self.0.wrapping_byte_add(add)) }}
/// A pointer to an area in userspace memory, which can be either read-only or read-write.////// All methods on this struct are safe: attempting to read or write on bad addresses (either out of/// the bound of the slice or unmapped addresses) will return [`EFAULT`]. Concurrent access,/// *including data races to/from userspace memory*, is permitted, because fundamentally another/// userspace thread/process could always be modifying memory at the same time (in the same way that/// userspace Rust's [`std::io`] permits data races with the contents of files on disk). In the/// presence of a race, the exact byte values read/written are unspecified but the operation is/// well-defined. Kernelspace code should validate its copy of data after completing a read, and not/// expect that multiple reads of the same address will return the same value.////// These APIs are designed to make it difficult to accidentally write TOCTOU (time-of-check to/// time-of-use) bugs. Every time a memory location is read, the reader's position is advanced by/// the read length and the next read will start from there. This helps prevent accidentally reading/// the same location twice and causing a TOCTOU bug.////// Creating a [`UserSliceReader`] and/or [`UserSliceWriter`] consumes the `UserSlice`, helping/// ensure that there aren't multiple readers or writers to the same location.////// If double-fetching a memory location is necessary for some reason, then that is done by creating/// multiple readers to the same memory location, e.g. using [`clone_reader`].////// # Examples////// Takes a region of userspace memory from the current process, and modify it by adding one to/// every byte in the region.////// ```no_run/// use kernel::ffi::c_void;/// use kernel::uaccess::{UserPtr, UserSlice};////// fn bytes_add_one(uptr: UserPtr, len: usize) -> Result {/// let (read, mut write) = UserSlice::new(uptr, len).reader_writer();////// let mut buf = KVec::new();/// read.read_all(&mut buf, GFP_KERNEL)?;////// for b in &mut buf {/// *b = b.wrapping_add(1);/// }////// write.write_slice(&buf)?;/// Ok(())/// }/// ```////// Example illustrating a TOCTOU (time-of-check to time-of-use) bug.////// ```no_run/// use kernel::ffi::c_void;/// use kernel::uaccess::{UserPtr, UserSlice};////// /// Returns whether the data in this region is valid./// fn is_valid(uptr: UserPtr, len: usize) -> Result<bool> {/// let read = UserSlice::new(uptr, len).reader();////// let mut buf = KVec::new();/// read.read_all(&mut buf, GFP_KERNEL)?;////// todo!()/// }////// /// Returns the bytes behind this user pointer if they are valid./// fn get_bytes_if_valid(uptr: UserPtr, len: usize) -> Result<KVec<u8>> {/// if !is_valid(uptr, len)? {/// return Err(EINVAL);/// }////// let read = UserSlice::new(uptr, len).reader();////// let mut buf = KVec::new();/// read.read_all(&mut buf, GFP_KERNEL)?;////// // THIS IS A BUG! The bytes could have changed since we checked them./// ///// // To avoid this kind of bug, don't call `UserSlice::new` multiple/// // times with the same address./// Ok(buf)/// }/// ```////// [`std::io`]: https://doc.rust-lang.org/std/io/index.html/// [`clone_reader`]: UserSliceReader::clone_readerpub struct UserSlice { ptr: UserPtr, length: usize,}
impl UserSlice { /// Constructs a user slice from a raw pointer and a length in bytes. /// /// Constructing a [`UserSlice`] performs no checks on the provided address and length, it can /// safely be constructed inside a kernel thread with no current userspace process. Reads and /// writes wrap the kernel APIs `copy_from_user` and `copy_to_user`, which check the memory map /// of the current process and enforce that the address range is within the user range (no /// additional calls to `access_ok` are needed). Validity of the pointer is checked when you /// attempt to read or write, not in the call to `UserSlice::new`. /// /// Callers must be careful to avoid time-of-check-time-of-use (TOCTOU) issues. The simplest way /// is to create a single instance of [`UserSlice`] per user memory block as it reads each byte /// at most once. pub fn new(ptr: UserPtr, length: usize) -> Self { UserSlice { ptr, length } }
/// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. pub fn read_all<A: Allocator>(self, buf: &mut Vec<u8, A>, flags: Flags) -> Result { self.reader().read_all(buf, flags) }
/// Constructs a [`UserSliceReader`]. pub fn reader(self) -> UserSliceReader { UserSliceReader { ptr: self.ptr, length: self.length, } }
/// Constructs a [`UserSliceWriter`]. pub fn writer(self) -> UserSliceWriter { UserSliceWriter { ptr: self.ptr, length: self.length, } }
/// Constructs both a [`UserSliceReader`] and a [`UserSliceWriter`]. /// /// Usually when this is used, you will first read the data, and then overwrite it afterwards. pub fn reader_writer(self) -> (UserSliceReader, UserSliceWriter) { ( UserSliceReader { ptr: self.ptr, length: self.length, }, UserSliceWriter { ptr: self.ptr, length: self.length, }, ) }}
/// A reader for [`UserSlice`].////// Used to incrementally read from the user slice.pub struct UserSliceReader { ptr: UserPtr, length: usize,}
impl UserSliceReader { /// Skip the provided number of bytes. /// /// Returns an error if skipping more than the length of the buffer. pub fn skip(&mut self, num_skip: usize) -> Result { // Update `self.length` first since that's the fallible part of this operation. self.length = self.length.checked_sub(num_skip).ok_or(EFAULT)?; self.ptr = self.ptr.wrapping_byte_add(num_skip); Ok(()) }
/// Create a reader that can access the same range of data. /// /// Reading from the clone does not advance the current reader. /// /// The caller should take care to not introduce TOCTOU issues, as described in the /// documentation for [`UserSlice`]. pub fn clone_reader(&self) -> UserSliceReader { UserSliceReader { ptr: self.ptr, length: self.length, } }
/// Returns the number of bytes left to be read from this reader. /// /// Note that even reading less than this number of bytes may fail. pub fn len(&self) -> usize { self.length }
/// Returns `true` if no data is available in the io buffer. pub fn is_empty(&self) -> bool { self.length == 0 }
/// Reads raw data from the user slice into a kernel buffer. /// /// For a version that uses `&mut [u8]`, please see [`UserSliceReader::read_slice`]. /// /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error. /// /// # Guarantees /// /// After a successful call to this method, all bytes in `out` are initialized. pub fn read_raw(&mut self, out: &mut [MaybeUninit<u8>]) -> Result { let len = out.len(); let out_ptr = out.as_mut_ptr().cast::<c_void>(); if len > self.length { return Err(EFAULT); } // SAFETY: `out_ptr` points into a mutable slice of length `len`, so we may write // that many bytes to it. let res = unsafe { bindings::copy_from_user(out_ptr, self.ptr.as_const_ptr(), len) }; if res != 0 { return Err(EFAULT); } self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; Ok(()) }
/// Reads raw data from the user slice into a kernel buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of /// bounds of this [`UserSliceReader`]. This call may modify `out` even if it returns an error. pub fn read_slice(&mut self, out: &mut [u8]) -> Result { // SAFETY: The types are compatible and `read_raw` doesn't write uninitialized bytes to // `out`. let out = unsafe { &mut *(core::ptr::from_mut(out) as *mut [MaybeUninit<u8>]) }; self.read_raw(out) }
/// Reads raw data from the user slice into a kernel buffer partially. /// /// This is the same as [`Self::read_slice`] but considers the given `offset` into `out` and /// truncates the read to the boundaries of `self` and `out`. /// /// On success, returns the number of bytes read. pub fn read_slice_partial(&mut self, out: &mut [u8], offset: usize) -> Result<usize> { let end = offset.saturating_add(self.len()).min(out.len());
let Some(dst) = out.get_mut(offset..end) else { return Ok(0); };
self.read_slice(dst)?; Ok(dst.len()) }
/// Reads raw data from the user slice into a kernel buffer partially. /// /// This is the same as [`Self::read_slice_partial`] but updates the given [`file::Offset`] by /// the number of bytes read. /// /// This is equivalent to C's `simple_write_to_buffer()`. /// /// On success, returns the number of bytes read. pub fn read_slice_file(&mut self, out: &mut [u8], offset: &mut file::Offset) -> Result<usize> { if offset.is_negative() { return Err(EINVAL); }
let Ok(offset_index) = (*offset).try_into() else { return Ok(0); };
let read = self.read_slice_partial(out, offset_index)?;
// OVERFLOW: `offset + read <= data.len() <= isize::MAX <= Offset::MAX` *offset += read as i64;
Ok(read) }
/// Reads a value of the specified type. /// /// Fails with [`EFAULT`] if the read happens on a bad address, or if the read goes out of /// bounds of this [`UserSliceReader`]. pub fn read<T: FromBytes>(&mut self) -> Result<T> { let len = size_of::<T>(); if len > self.length { return Err(EFAULT); } let mut out: MaybeUninit<T> = MaybeUninit::uninit(); // SAFETY: The local variable `out` is valid for writing `size_of::<T>()` bytes. // // By using the _copy_from_user variant, we skip the check_object_size check that verifies // the kernel pointer. This mirrors the logic on the C side that skips the check when the // length is a compile-time constant. let res = unsafe { bindings::_copy_from_user( out.as_mut_ptr().cast::<c_void>(), self.ptr.as_const_ptr(), len, ) }; if res != 0 { return Err(EFAULT); } self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; // SAFETY: The read above has initialized all bytes in `out`, and since `T` implements // `FromBytes`, any bit-pattern is a valid value for this type. Ok(unsafe { out.assume_init() }) }
/// Reads the entirety of the user slice, appending it to the end of the provided buffer. /// /// Fails with [`EFAULT`] if the read happens on a bad address. pub fn read_all<A: Allocator>(mut self, buf: &mut Vec<u8, A>, flags: Flags) -> Result { let len = self.length; buf.reserve(len, flags)?;
// The call to `reserve` was successful, so the spare capacity is at least `len` bytes long. self.read_raw(&mut buf.spare_capacity_mut()[..len])?;
// SAFETY: Since the call to `read_raw` was successful, so the next `len` bytes of the // vector have been initialized. unsafe { buf.inc_len(len) }; Ok(()) }
/// Read a NUL-terminated string from userspace and return it. /// /// The string is read into `buf` and a NUL-terminator is added if the end of `buf` is reached. /// Since there must be space to add a NUL-terminator, the buffer must not be empty. The /// returned `&CStr` points into `buf`. /// /// Fails with [`EFAULT`] if the read happens on a bad address (some data may have been /// copied). #[doc(alias = "strncpy_from_user")] pub fn strcpy_into_buf<'buf>(self, buf: &'buf mut [u8]) -> Result<&'buf CStr> { if buf.is_empty() { return Err(EINVAL); }
// SAFETY: The types are compatible and `strncpy_from_user` doesn't write uninitialized // bytes to `buf`. let mut dst = unsafe { &mut *(core::ptr::from_mut(buf) as *mut [MaybeUninit<u8>]) };
// We never read more than `self.length` bytes. if dst.len() > self.length { dst = &mut dst[..self.length]; }
let mut len = raw_strncpy_from_user(dst, self.ptr)?; if len < dst.len() { // Add one to include the NUL-terminator. len += 1; } else if len < buf.len() { // This implies that `len == dst.len() < buf.len()`. // // This means that we could not fill the entire buffer, but we had to stop reading // because we hit the `self.length` limit of this `UserSliceReader`. Since we did not // fill the buffer, we treat this case as if we tried to read past the `self.length` // limit and received a page fault, which is consistent with other `UserSliceReader` // methods that also return page faults when you exceed `self.length`. return Err(EFAULT); } else { // This implies that `len == buf.len()`. // // This means that we filled the buffer exactly. In this case, we add a NUL-terminator // and return it. Unlike the `len < dst.len()` branch, don't modify `len` because it // already represents the length including the NUL-terminator. // // SAFETY: Due to the check at the beginning, the buffer is not empty. unsafe { *buf.last_mut().unwrap_unchecked() = 0 }; }
// This method consumes `self`, so it can only be called once, thus we do not need to // update `self.length`. This sidesteps concerns such as whether `self.length` should be // incremented by `len` or `len-1` in the `len == buf.len()` case.
// SAFETY: There are two cases: // * If we hit the `len < dst.len()` case, then `raw_strncpy_from_user` guarantees that // this slice contains exactly one NUL byte at the end of the string. // * Otherwise, `raw_strncpy_from_user` guarantees that the string contained no NUL bytes, // and we have since added a NUL byte at the end. Ok(unsafe { CStr::from_bytes_with_nul_unchecked(&buf[..len]) }) }}
/// A writer for [`UserSlice`].////// Used to incrementally write into the user slice.pub struct UserSliceWriter { ptr: UserPtr, length: usize,}
impl UserSliceWriter { /// Returns the amount of space remaining in this buffer. /// /// Note that even writing less than this number of bytes may fail. pub fn len(&self) -> usize { self.length }
/// Returns `true` if no more data can be written to this buffer. pub fn is_empty(&self) -> bool { self.length == 0 }
/// Low-level write from a raw pointer. /// /// # Safety /// /// The caller must ensure that `from` is valid for reads of `len` bytes. unsafe fn write_raw(&mut self, from: *const u8, len: usize) -> Result { if len > self.length { return Err(EFAULT); }
// SAFETY: Caller guarantees `from` is valid for `len` bytes (see this function's // safety contract). let res = unsafe { bindings::copy_to_user(self.ptr.as_mut_ptr(), from.cast(), len) }; if res != 0 { return Err(EFAULT); } self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; Ok(()) }
/// Writes raw data to this user pointer from a kernel buffer. /// /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even /// if it returns an error. pub fn write_slice(&mut self, data: &[u8]) -> Result { // SAFETY: `data` is a valid slice, so `data.as_ptr()` is valid for // reading `data.len()` bytes. unsafe { self.write_raw(data.as_ptr(), data.len()) } }
/// Writes raw data to this user pointer from a DMA coherent allocation. /// /// Copies `count` bytes from `alloc` starting from `offset` into this userspace slice. /// /// # Errors /// /// - [`EOVERFLOW`]: `offset + count` overflows. /// - [`ERANGE`]: `offset + count` exceeds the size of `alloc`, or `count` exceeds the /// size of the user-space buffer. /// - [`EFAULT`]: the write hits a bad address or goes out of bounds of this /// [`UserSliceWriter`]. /// /// This call may modify the associated userspace slice even if it returns an error. /// /// Note: The memory may be concurrently modified by hardware (e.g., DMA). In such cases, /// the copied data may be inconsistent, but this does not cause undefined behavior. /// /// # Example /// /// Copy the first 256 bytes of a DMA coherent allocation into a userspace buffer: /// /// ```no_run /// use kernel::uaccess::UserSliceWriter; /// use kernel::dma::Coherent; /// /// fn copy_dma_to_user( /// mut writer: UserSliceWriter, /// alloc: &Coherent<'_, [u8]>, /// ) -> Result { /// writer.write_dma(alloc, 0, 256) /// } /// ``` pub fn write_dma<T: KnownSize + AsBytes + ?Sized>( &mut self, alloc: &Coherent<'_, T>, offset: usize, count: usize, ) -> Result { let len = alloc.size(); if offset.checked_add(count).ok_or(EOVERFLOW)? > len { return Err(ERANGE); }
if count > self.len() { return Err(ERANGE); }
// SAFETY: `as_ptr()` returns a valid pointer to a memory region of `count()` bytes, as // guaranteed by the `Coherent` invariants. The check above ensures `offset + count <= len`. let src_ptr = unsafe { alloc.as_ptr().cast::<u8>().add(offset) };
// Note: Use `write_raw` instead of `write_slice` because the allocation is coherent // memory that hardware may modify (e.g., DMA); we cannot form a `&[u8]` slice over // such volatile memory. // // SAFETY: `src_ptr` points into the allocation and is valid for `count` bytes (see above). unsafe { self.write_raw(src_ptr, count) } }
/// Writes raw data to this user pointer from a kernel buffer partially. /// /// This is the same as [`Self::write_slice`] but considers the given `offset` into `data` and /// truncates the write to the boundaries of `self` and `data`. /// /// On success, returns the number of bytes written. #[inline] pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result<usize> { let end = offset.saturating_add(self.len()).min(data.len());
let Some(src) = data.get(offset..end) else { return Ok(0); };
self.write_slice(src)?; Ok(src.len()) }
/// Writes raw data to this user pointer from a kernel buffer partially. /// /// This is the same as [`Self::write_slice_partial`] but updates the given [`file::Offset`] by /// the number of bytes written. /// /// This is equivalent to C's `simple_read_from_buffer()`. /// /// On success, returns the number of bytes written. pub fn write_slice_file(&mut self, data: &[u8], offset: &mut file::Offset) -> Result<usize> { if offset.is_negative() { return Err(EINVAL); }
let Ok(offset_index) = (*offset).try_into() else { return Ok(0); };
let written = self.write_slice_partial(data, offset_index)?;
// OVERFLOW: `offset + written <= data.len() <= isize::MAX <= Offset::MAX` *offset += written as i64;
Ok(written) }
/// Writes the provided Rust value to this userspace pointer. /// /// Fails with [`EFAULT`] if the write happens on a bad address, or if the write goes out of /// bounds of this [`UserSliceWriter`]. This call may modify the associated userspace slice even /// if it returns an error. pub fn write<T: AsBytes>(&mut self, value: &T) -> Result { let len = size_of::<T>(); if len > self.length { return Err(EFAULT); } // SAFETY: The reference points to a value of type `T`, so it is valid for reading // `size_of::<T>()` bytes. // // By using the _copy_to_user variant, we skip the check_object_size check that verifies the // kernel pointer. This mirrors the logic on the C side that skips the check when the length // is a compile-time constant. let res = unsafe { bindings::_copy_to_user( self.ptr.as_mut_ptr(), core::ptr::from_ref(value).cast::<c_void>(), len, ) }; if res != 0 { return Err(EFAULT); } self.ptr = self.ptr.wrapping_byte_add(len); self.length -= len; Ok(()) }
/// Writes as much of the provided value as fits in the remaining buffer. /// /// Copies `min(size_of::<T>(), self.len())` bytes to userspace. Returns the number of bytes /// actually written. This is useful for versioned structs where an older userspace may provide /// a smaller buffer than the current kernel struct. /// /// Fails with [`EFAULT`] if the write happens on a bad address. This call may modify the /// associated userspace slice even if it returns an error. #[inline] pub fn write_truncated<T: AsBytes>(&mut self, value: &T) -> Result<usize> { self.write_slice_partial(value.as_bytes(), 0) }}
/// Reads a nul-terminated string into `dst` and returns the length.////// This reads from userspace until a NUL byte is encountered, or until `dst.len()` bytes have been/// read. Fails with [`EFAULT`] if a read happens on a bad address (some data may have been/// copied). When the end of the buffer is encountered, no NUL byte is added, so the string is/// *not* guaranteed to be NUL-terminated when `Ok(dst.len())` is returned.////// # Guarantees////// When this function returns `Ok(len)`, it is guaranteed that the first `len` bytes of `dst` are/// initialized and non-zero. Furthermore, if `len < dst.len()`, then `dst[len]` is a NUL byte.#[inline]fn raw_strncpy_from_user(dst: &mut [MaybeUninit<u8>], src: UserPtr) -> Result<usize> { // CAST: Slice lengths are guaranteed to be `<= isize::MAX`. let len = dst.len() as isize;
// SAFETY: `dst` is valid for writing `dst.len()` bytes. let res = unsafe { bindings::strncpy_from_user( dst.as_mut_ptr().cast::<c_char>(), src.as_const_ptr().cast::<c_char>(), len, ) };
if res < 0 { return Err(Error::from_errno(res as i32)); }
#[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] assert!(res <= len);
// GUARANTEES: `strncpy_from_user` was successful, so `dst` has contents in accordance with the // guarantees of this function. Ok(res as usize)}