Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430// SPDX-License-Identifier: GPL-2.0 OR MIT
use super::*;
/// The actual data that gets threaded through the callbacks.struct SmData<'a, 'ctx, T: DriverGpuVm> { gpuvm: &'a mut UniqueRefGpuVm<T>, user_context: &'a mut T::SmContext<'ctx>,}
/// Adds an extra field to `SmData` for `sm_map()` callbacks.////// # Invariants////// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`.#[repr(C)]struct SmMapData<'a, 'ctx, T: DriverGpuVm> { sm_data: SmData<'a, 'ctx, T>, vm_bo: &'a GpuVmBo<T>,}
/// The argument for [`UniqueRefGpuVm::sm_map`].pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> { /// Address in GPU virtual address space. pub addr: u64, /// Length of mapping to create. pub range: u64, /// Offset in GEM object. pub gem_offset: u64, /// The GEM object to map. pub vm_bo: &'a GpuVmBo<T>, /// The user-provided context type. pub context: &'a mut T::SmContext<'ctx>,}
impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> { fn raw_request(&self) -> bindings::drm_gpuvm_map_req { bindings::drm_gpuvm_map_req { map: bindings::drm_gpuva_op_map { va: bindings::drm_gpuva_op_map__bindgen_ty_1 { addr: self.addr, range: self.range, }, gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { offset: self.gem_offset, obj: self.vm_bo.obj().as_raw(), }, }, } }}
/// Represents an `sm_step_map` operation that has not yet been completed.pub struct OpMap<'op, T: DriverGpuVm> { op: &'op bindings::drm_gpuva_op_map, // Since these abstractions are designed for immediate mode, the VM BO needs to be // pre-allocated, so we always have it available when we reach this point. vm_bo: &'op GpuVmBo<T>, // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not // coerce to `OpMap<'short, T>`. This ensures that the user can't return // the wrong `OpMapped` value. _invariant: PhantomData<*mut &'op mut T>,}
impl<'op, T: DriverGpuVm> OpMap<'op, T> { /// The base address of the new mapping. pub fn addr(&self) -> u64 { self.op.va.addr }
/// The length of the new mapping. pub fn length(&self) -> u64 { self.op.va.range }
/// The offset within the [`drm_gem_object`](DriverGpuVm::Object). pub fn gem_offset(&self) -> u64 { self.op.gem.offset }
/// The [`drm_gem_object`](DriverGpuVm::Object) to map. pub fn obj(&self) -> &T::Object { // SAFETY: The `obj` pointer is guaranteed to be valid. unsafe { <T::Object as IntoGEMObject>::from_raw(self.op.gem.obj) } }
/// The [`GpuVmBo`] that the new VA will be associated with. pub fn vm_bo(&self) -> &GpuVmBo<T> { self.vm_bo }
/// Use the pre-allocated VA to carry out this map operation. pub fn insert(self, va: GpuVaAlloc<T>, va_data: impl PinInit<T::VaData>) -> OpMapped<'op, T> { let va = va.prepare(va_data); // SAFETY: By the type invariants we may access the interval tree. unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) };
let _gpuva_guard = self.vm_bo().lock_gpuva(); // SAFETY: The va is prepared for insertion, and we hold the GEM lock. unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) };
OpMapped { _invariant: self._invariant, } }}
/// Represents a completed [`OpMap`] operation.pub struct OpMapped<'op, T> { _invariant: PhantomData<*mut &'op mut T>,}
/// Represents an `sm_step_unmap` operation that has not yet been completed.pub struct OpUnmap<'op, T: DriverGpuVm> { op: &'op bindings::drm_gpuva_op_unmap, // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the // wrong`OpUnmapped` value. _invariant: PhantomData<*mut &'op mut T>,}
impl<'op, T: DriverGpuVm> OpUnmap<'op, T> { /// Indicates whether this [`GpuVa`] is physically contiguous with the /// original mapping request. /// /// Optionally, if `keep` is set, drivers may keep the actual page table /// mappings for this `drm_gpuva`, adding the missing page table entries /// only and update the `drm_gpuvm` accordingly. pub fn keep(&self) -> bool { self.op.keep }
/// The range being unmapped. pub fn va(&self) -> &GpuVa<T> { // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, // and it's not sparse by the `GpuVm<T>` type invariants. unsafe { GpuVa::<T>::from_raw(self.op.va) } }
/// Remove the VA. pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved<T>) { // SAFETY: The op references a valid drm_gpuva in the GPUVM. unsafe { bindings::drm_gpuva_unmap(self.op) }; // SAFETY: The va is no longer in the interval tree so we may unlink it. unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) };
// SAFETY: We just removed this va from the `GpuVm<T>`. let va = unsafe { GpuVaRemoved::from_raw(self.op.va) };
( OpUnmapped { _invariant: self._invariant, }, va, ) }}
/// Represents a completed [`OpUnmap`] operation.pub struct OpUnmapped<'op, T> { _invariant: PhantomData<*mut &'op mut T>,}
/// Represents an `sm_step_remap` operation that has not yet been completed.pub struct OpRemap<'op, T: DriverGpuVm> { op: &'op bindings::drm_gpuva_op_remap, // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the // wrong`OpRemapped` value. _invariant: PhantomData<*mut &'op mut T>,}
impl<'op, T: DriverGpuVm> OpRemap<'op, T> { /// The preceding part of a split mapping. #[inline] pub fn prev(&self) -> Option<&OpRemapMapData> { // SAFETY: We checked for null, so the pointer must be valid. NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) }
/// The subsequent part of a split mapping. #[inline] pub fn next(&self) -> Option<&OpRemapMapData> { // SAFETY: We checked for null, so the pointer must be valid. NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) }) }
/// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original /// mapping request. /// /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm` /// accordingly. #[inline] pub fn keep(&self) -> bool { // SAFETY: The unmap pointer is always valid. unsafe { (*self.op.unmap).keep } }
/// The range being unmapped. #[inline] pub fn va_to_unmap(&self) -> &GpuVa<T> { // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it, // and it's not sparse by the `GpuVm<T>` type invariants. unsafe { GpuVa::<T>::from_raw((*self.op.unmap).va) } }
/// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped. #[inline] pub fn obj(&self) -> &T::Object { self.va_to_unmap().obj() }
/// The [`GpuVmBo`] that is being remapped. #[inline] pub fn vm_bo(&self) -> &GpuVmBo<T> { self.va_to_unmap().vm_bo() }
/// Update the GPUVM to perform the remapping. pub fn remap( self, va_alloc: [GpuVaAlloc<T>; 2], prev_data: impl PinInit<T::VaData>, next_data: impl PinInit<T::VaData>, ) -> (OpRemapped<'op, T>, OpRemapRet<T>) { let [va1, va2] = va_alloc;
let mut unused_va = None; let mut prev_ptr = ptr::null_mut(); let mut next_ptr = ptr::null_mut(); if self.prev().is_some() { prev_ptr = va1.prepare(prev_data); } else { unused_va = Some(va1); } if self.next().is_some() { next_ptr = va2.prepare(next_data); } else { unused_va = Some(va2); }
// SAFETY: the pointers are non-null when required unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) };
let gpuva_guard = self.vm_bo().lock_gpuva(); if !prev_ptr.is_null() { // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) }; } if !next_ptr.is_null() { // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo. unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) }; } drop(gpuva_guard);
// SAFETY: The va is no longer in the interval tree so we may unlink it. unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) };
( OpRemapped { _invariant: self._invariant, }, OpRemapRet { // SAFETY: We just removed this va from the `GpuVm<T>`. unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) }, unused_va, }, ) }}
/// Part of an [`OpRemap`] that represents a new mapping.#[repr(transparent)]pub struct OpRemapMapData(bindings::drm_gpuva_op_map);
impl OpRemapMapData { /// # Safety /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`. unsafe fn from_raw<'a>(ptr: NonNull<bindings::drm_gpuva_op_map>) -> &'a Self { // SAFETY: ok per safety requirements unsafe { ptr.cast().as_ref() } }
/// The base address of the new mapping. pub fn addr(&self) -> u64 { self.0.va.addr }
/// The length of the new mapping. pub fn length(&self) -> u64 { self.0.va.range }
/// The offset within the [`drm_gem_object`](DriverGpuVm::Object). pub fn gem_offset(&self) -> u64 { self.0.gem.offset }}
/// Struct containing objects removed or not used by [`OpRemap::remap`].pub struct OpRemapRet<T: DriverGpuVm> { /// The `drm_gpuva` that was removed. pub unmapped_va: GpuVaRemoved<T>, /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is /// returned here. pub unused_va: Option<GpuVaAlloc<T>>,}
/// Represents a completed [`OpRemap`] operation.pub struct OpRemapped<'op, T> { _invariant: PhantomData<*mut &'op mut T>,}
impl<T: DriverGpuVm> UniqueRefGpuVm<T> { /// Create a mapping, removing or remapping anything that overlaps. /// /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping. #[inline] pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result { if req.vm_bo.gpuvm() != &**self { return Err(EINVAL); }
let gpuvm = self.as_raw(); let raw_req = req.raw_request(); // INVARIANT: Checked above that `vm_bo.gpuvm() == self`. let mut p = SmMapData { sm_data: SmData { gpuvm: self, user_context: req.context, }, vm_bo: req.vm_bo, }; // SAFETY: // * raw_request() creates a valid request. // * The private data is valid to be interpreted as both SmData and SmMapData since the // first field of SmMapData is SmData. to_result(unsafe { bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req) }) }
/// Remove any mappings in the given region. /// /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range. #[inline] pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result { let gpuvm = self.as_raw(); let mut p = SmData { gpuvm: self, user_context: context, }; // SAFETY: // * raw_request() creates a valid request. // * The private data is a valid SmData. to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) }) }}
impl<T: DriverGpuVm> GpuVm<T> { /// # Safety /// Must be called from `sm_map` with a pointer to `SmMapData`. pub(super) unsafe extern "C" fn sm_step_map( op: *mut bindings::drm_gpuva_op, p: *mut c_void, ) -> c_int { // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes // an `SmMapData` as private data. let p = unsafe { &mut *p.cast::<SmMapData<'_, '_, T>>() }; let op = OpMap { // SAFETY: sm_step_map is called with a map operation. op: unsafe { &(*op).__bindgen_anon_1.map }, vm_bo: p.vm_bo, _invariant: PhantomData, }; match p .sm_data .gpuvm .data() .sm_step_map(op, p.sm_data.user_context) { Ok(OpMapped { .. }) => 0, Err(err) => err.to_errno(), } }
/// # Safety /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. pub(super) unsafe extern "C" fn sm_step_unmap( op: *mut bindings::drm_gpuva_op, p: *mut c_void, ) -> c_int { // SAFETY: The caller provides a pointer that can be treated as `SmData`. let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() }; let op = OpUnmap { // SAFETY: sm_step_unmap is called with an unmap operation. op: unsafe { &(*op).__bindgen_anon_1.unmap }, _invariant: PhantomData, }; match p.gpuvm.data().sm_step_unmap(op, p.user_context) { Ok(OpUnmapped { .. }) => 0, Err(err) => err.to_errno(), } }
/// # Safety /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`. pub(super) unsafe extern "C" fn sm_step_remap( op: *mut bindings::drm_gpuva_op, p: *mut c_void, ) -> c_int { // SAFETY: The caller provides a pointer that can be treated as `SmData`. let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() }; let op = OpRemap { // SAFETY: sm_step_remap is called with a remap operation. op: unsafe { &(*op).__bindgen_anon_1.remap }, _invariant: PhantomData, }; match p.gpuvm.data().sm_step_remap(op, p.user_context) { Ok(OpRemapped { .. }) => 0, Err(err) => err.to_errno(), } }}