Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383// SPDX-License-Identifier: GPL-2.0
//! Kernel page allocation and management.
use crate::{ alloc::{ AllocError, Flags, // }, bindings, error::{ code::*, Result, // }, uaccess::UserSliceReader, //};use core::{ marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::{ self, NonNull, // }, //};
/// A bitwise shift for the page size.pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;
/// The number of bytes in a page.pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;
/// A bitmask that gives the page containing a given address.pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);
/// Rounds up to the next multiple of [`PAGE_SIZE`].////// Returns [`None`] on integer overflow.////// # Examples////// ```/// use kernel::page::{/// page_align,/// PAGE_SIZE,/// };////// // Requested address is already aligned./// assert_eq!(page_align(0x0), Some(0x0));/// assert_eq!(page_align(PAGE_SIZE), Some(PAGE_SIZE));////// // Requested address needs alignment up./// assert_eq!(page_align(0x1), Some(PAGE_SIZE));/// assert_eq!(page_align(PAGE_SIZE + 1), Some(2 * PAGE_SIZE));////// // Requested address causes overflow (returns `None`)./// let overflow_addr = usize::MAX - (PAGE_SIZE / 2);/// assert_eq!(page_align(overflow_addr), None);/// ```#[inline(always)]pub const fn page_align(addr: usize) -> Option<usize> { let Some(sum) = addr.checked_add(PAGE_SIZE - 1) else { return None; }; Some(sum & PAGE_MASK)}
/// Representation of a non-owning reference to a [`Page`].////// This type provides a borrowed version of a [`Page`] that is owned by some other entity, e.g. a/// [`Vmalloc`] allocation such as [`VBox`].////// # Example////// ```/// # use kernel::{bindings, prelude::*};/// use kernel::page::{BorrowedPage, Page, PAGE_SIZE};/// # use core::{mem::MaybeUninit, ptr, ptr::NonNull };////// fn borrow_page<'a>(vbox: &'a mut VBox<MaybeUninit<[u8; PAGE_SIZE]>>) -> BorrowedPage<'a> {/// let ptr = ptr::from_ref(&**vbox);////// // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory./// let page = unsafe { bindings::vmalloc_to_page(ptr.cast()) };////// // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid/// // pointer to `Vmalloc` memory./// let page = unsafe { NonNull::new_unchecked(page) };////// // SAFETY:/// // - `self.0` is a valid pointer to a `struct page`./// // - `self.0` is valid for the entire lifetime of `self`./// unsafe { BorrowedPage::from_raw(page) }/// }////// let mut vbox = VBox::<[u8; PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;/// let page = borrow_page(&mut vbox);////// // SAFETY: There is no concurrent read or write to this page./// unsafe { page.fill_zero_raw(0, PAGE_SIZE)? };/// # Ok::<(), Error>(())/// ```////// # Invariants////// The borrowed underlying pointer to a `struct page` is valid for the entire lifetime `'a`.////// [`VBox`]: kernel::alloc::VBox/// [`Vmalloc`]: kernel::alloc::allocator::Vmallocpub struct BorrowedPage<'a>(ManuallyDrop<Page>, PhantomData<&'a Page>);
impl<'a> BorrowedPage<'a> { /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. /// /// # Safety /// /// - `ptr` must point to a valid `bindings::page`. /// - `ptr` must remain valid for the entire lifetime `'a`. pub unsafe fn from_raw(ptr: NonNull<bindings::page>) -> Self { let page = Page { page: ptr };
// INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime // `'a`. Self(ManuallyDrop::new(page), PhantomData) }}
impl<'a> Deref for BorrowedPage<'a> { type Target = Page;
fn deref(&self) -> &Self::Target { &self.0 }}
/// Trait to be implemented by types which provide an [`Iterator`] implementation of/// [`BorrowedPage`] items, such as [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter).pub trait AsPageIter { /// The [`Iterator`] type, e.g. [`VmallocPageIter`](kernel::alloc::allocator::VmallocPageIter). type Iter<'a>: Iterator<Item = BorrowedPage<'a>> where Self: 'a;
/// Returns an [`Iterator`] of [`BorrowedPage`] items over all pages owned by `self`. fn page_iter(&mut self) -> Self::Iter<'_>;}
/// A pointer to a page that owns the page allocation.////// # Invariants////// The pointer is valid, and has ownership over the page.pub struct Page { page: NonNull<bindings::page>,}
// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across// threads is safe.unsafe impl Send for Page {}
// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing// them concurrently is safe.unsafe impl Sync for Page {}
impl Page { /// Allocates a new page. /// /// # Examples /// /// Allocate memory for a page. /// /// ``` /// use kernel::page::Page; /// /// let page = Page::alloc_page(GFP_KERNEL)?; /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` /// /// Allocate memory for a page and zero its contents. /// /// ``` /// use kernel::page::Page; /// /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` #[inline] pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it // is always safe to call this method. let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; let page = NonNull::new(page).ok_or(AllocError)?; // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly // allocated page. We transfer that ownership to the new `Page` object. Ok(Self { page }) }
/// Returns a raw pointer to the page. pub fn as_ptr(&self) -> *mut bindings::page { self.page.as_ptr() }
/// Get the node id containing this page. #[inline] pub fn nid(&self) -> i32 { // SAFETY: Always safe to call with a valid page. unsafe { bindings::page_to_nid(self.as_ptr()) } }
/// Runs a piece of code with this page mapped to an address. /// /// The page is unmapped when this call returns. /// /// # Using the raw pointer /// /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might /// only be mapped on the current thread, and when that is the case, dereferencing it on other /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't /// cause data races, the memory may be uninitialized, and so on. /// /// If multiple threads map the same page at the same time, then they may reference with /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { // SAFETY: `page` is valid due to the type invariants on `Page`. let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };
let res = f(mapped_addr.cast());
// This unmaps the page mapped above. // // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. // // In other words, if this call to `kunmap_local` happens when a different page should be // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the // call just above in `with_page_mapped` that made that possible. In this case, it is the // unsafe block that wraps that other call that is incorrect. unsafe { bindings::kunmap_local(mapped_addr) };
res }
/// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. /// /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on /// this task, as this method uses a local mapping. /// /// If `off` and `len` refers to a region outside of this page, then this method returns /// [`EINVAL`] and does not call `f`. /// /// # Using the raw pointer /// /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for /// `len` bytes and for the duration in which the closure is called. The pointer might only be /// mapped on the current thread, and when that is the case, dereferencing it on other threads /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause /// data races, the memory may be uninitialized, and so on. /// /// If multiple threads map the same page at the same time, then they may reference with /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). fn with_pointer_into_page<T>( &self, off: usize, len: usize, f: impl FnOnce(*mut u8) -> Result<T>, ) -> Result<T> { let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;
if bounds_ok { self.with_page_mapped(move |page_addr| { // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will // result in a pointer that is in bounds or one off the end of the page. f(unsafe { page_addr.add(off) }) }) } else { Err(EINVAL) } }
/// Maps the page and reads from it into the given buffer. /// /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes /// outside of the page, then this call returns [`EINVAL`]. /// /// # Safety /// /// * Callers must ensure that `dst` is valid for writing `len` bytes. /// * Callers must ensure that this call does not race with a write to the same page that /// overlaps with this read. pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { self.with_pointer_into_page(offset, len, move |src| { // SAFETY: If `with_pointer_into_page` calls into this closure, then // it has performed a bounds check and guarantees that `src` is // valid for `len` bytes. // // There caller guarantees that there is no data race. unsafe { ptr::copy_nonoverlapping(src, dst, len) }; Ok(()) }) }
/// Maps the page and writes into it from the given buffer. /// /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes /// outside of the page, then this call returns [`EINVAL`]. /// /// # Safety /// /// * Callers must ensure that `src` is valid for reading `len` bytes. /// * Callers must ensure that this call does not race with a read or write to the same page /// that overlaps with this write. pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { self.with_pointer_into_page(offset, len, move |dst| { // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a // bounds check and guarantees that `dst` is valid for `len` bytes. // // There caller guarantees that there is no data race. unsafe { ptr::copy_nonoverlapping(src, dst, len) }; Ok(()) }) }
/// Maps the page and zeroes the given slice. /// /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes /// outside of the page, then this call returns [`EINVAL`]. /// /// # Safety /// /// Callers must ensure that this call does not race with a read or write to the same page that /// overlaps with this write. pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { self.with_pointer_into_page(offset, len, move |dst| { // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a // bounds check and guarantees that `dst` is valid for `len` bytes. // // There caller guarantees that there is no data race. unsafe { ptr::write_bytes(dst, 0u8, len) }; Ok(()) }) }
/// Copies data from userspace into this page. /// /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes /// outside of the page, then this call returns [`EINVAL`]. /// /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. /// However, they are not allowed on the page you are copying into. /// /// # Safety /// /// Callers must ensure that this call does not race with a read or write to the same page that /// overlaps with this write. pub unsafe fn copy_from_user_slice_raw( &self, reader: &mut UserSliceReader, offset: usize, len: usize, ) -> Result { self.with_pointer_into_page(offset, len, move |dst| { // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have // exclusive access to the slice since the caller guarantees that there are no races. reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) }) }}
impl Drop for Page { #[inline] fn drop(&mut self) { // SAFETY: By the type invariants, we have ownership of the page and can free it. unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; }}