Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2025 Google LLC.
//! This module defines the `Thread` type, which represents a userspace thread that is using//! binder.//!//! The `Process` object stores all of the threads in an rb tree.
use kernel::{ bindings, bits::bit_u32, fs::LocalFile, list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, prelude::*, security, seq_file::SeqFile, seq_print, sync::atomic::{ordering::Relaxed, Atomic}, sync::{aref::ARef, Arc, CondVar, SpinLock}, task::Task, uaccess::{UserPtr, UserSlice, UserSliceReader}, uapi,};
use crate::{ allocation::{Allocation, AllocationView, BinderObject, BinderObjectRef, NewAllocation}, defs::*, error::BinderResult, process::{GetWorkOrRegister, Process}, ptr_align, stats::GLOBAL_STATS, trace::{trace_transaction_alloc_buf, trace_transaction_buffer_release}, transaction::{Transaction, TransactionFlag, TransactionFlags, TransactionInfo}, BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverCode, DeliverToRead,};
use core::mem::size_of;
fn is_aligned(value: usize, to: usize) -> bool { value % to == 0}
/// Stores the layout of the scatter-gather entries. This is used during the `translate_objects`/// call and is discarded when it returns.struct ScatterGatherState { /// A struct that tracks the amount of unused buffer space. unused_buffer_space: UnusedBufferSpace, /// Scatter-gather entries to copy. sg_entries: KVec<ScatterGatherEntry>, /// Indexes into `sg_entries` corresponding to the last binder_buffer_object that /// was processed and all of its ancestors. The array is in sorted order. ancestors: KVec<usize>,}
/// This entry specifies an additional buffer that should be copied using the scatter-gather/// mechanism.struct ScatterGatherEntry { /// The index in the offset array of the BINDER_TYPE_PTR that this entry originates from. obj_index: usize, /// Offset in target buffer. offset: usize, /// User address in source buffer. sender_uaddr: usize, /// Number of bytes to copy. length: usize, /// The minimum offset of the next fixup in this buffer. fixup_min_offset: usize, /// The offsets within this buffer that contain pointers which should be translated. pointer_fixups: KVec<PointerFixupEntry>,}
/// This entry specifies that a fixup should happen at `target_offset` of the/// buffer.enum PointerFixupEntry { /// A fixup for a `binder_buffer_object`. Fixup { /// The translated pointer to write. pointer_value: u64, /// The offset at which the value should be written. The offset is relative /// to the original buffer. target_offset: usize, }, /// A skip for a `binder_fd_array_object`. Skip { /// The number of bytes to skip. skip: usize, /// The offset at which the skip should happen. The offset is relative /// to the original buffer. target_offset: usize, },}
/// Return type of `apply_and_validate_fixup_in_parent`.struct ParentFixupInfo { /// The index of the parent buffer in `sg_entries`. parent_sg_index: usize, /// The number of ancestors of the buffer. /// /// The buffer is considered an ancestor of itself, so this is always at /// least one. num_ancestors: usize, /// New value of `fixup_min_offset` if this fixup is applied. new_min_offset: usize, /// The offset of the fixup in the target buffer. target_offset: usize,}
impl ScatterGatherState { /// Called when a `binder_buffer_object` or `binder_fd_array_object` tries /// to access a region in its parent buffer. These accesses have various /// restrictions, which this method verifies. /// /// The `parent_offset` and `length` arguments describe the offset and /// length of the access in the parent buffer. /// /// # Detailed restrictions /// /// Obviously the fixup must be in-bounds for the parent buffer. /// /// For safety reasons, we only allow fixups inside a buffer to happen /// at increasing offsets; additionally, we only allow fixup on the last /// buffer object that was verified, or one of its parents. /// /// Example of what is allowed: /// /// A /// B (parent = A, offset = 0) /// C (parent = A, offset = 16) /// D (parent = C, offset = 0) /// E (parent = A, offset = 32) // min_offset is 16 (C.parent_offset) /// /// Examples of what is not allowed: /// /// Decreasing offsets within the same parent: /// A /// C (parent = A, offset = 16) /// B (parent = A, offset = 0) // decreasing offset within A /// /// Arcerring to a parent that wasn't the last object or any of its parents: /// A /// B (parent = A, offset = 0) /// C (parent = A, offset = 0) /// C (parent = A, offset = 16) /// D (parent = B, offset = 0) // B is not A or any of A's parents fn validate_parent_fixup( &self, parent: usize, parent_offset: usize, length: usize, ) -> Result<ParentFixupInfo> { // Using `position` would also be correct, but `rposition` avoids // quadratic running times. let ancestors_i = self .ancestors .iter() .copied() .rposition(|sg_idx| self.sg_entries[sg_idx].obj_index == parent) .ok_or(EINVAL)?; let sg_idx = self.ancestors[ancestors_i]; let sg_entry = match self.sg_entries.get(sg_idx) { Some(sg_entry) => sg_entry, None => { pr_err_ratelimited!( "self.ancestors[{}] is {}, but self.sg_entries.len() is {}\n", ancestors_i, sg_idx, self.sg_entries.len() ); return Err(EINVAL); } }; if sg_entry.fixup_min_offset > parent_offset { pr_warn_ratelimited!( "validate_parent_fixup: fixup_min_offset={}, parent_offset={}\n", sg_entry.fixup_min_offset, parent_offset ); return Err(EINVAL); } let new_min_offset = parent_offset.checked_add(length).ok_or(EINVAL)?; if new_min_offset > sg_entry.length { pr_warn_ratelimited!( "validate_parent_fixup: new_min_offset={}, sg_entry.length={}\n", new_min_offset, sg_entry.length ); return Err(EINVAL); } let target_offset = sg_entry.offset.checked_add(parent_offset).ok_or(EINVAL)?; // The `ancestors_i + 1` operation can't overflow since the output of the addition is at // most `self.ancestors.len()`, which also fits in a usize. Ok(ParentFixupInfo { parent_sg_index: sg_idx, num_ancestors: ancestors_i + 1, new_min_offset, target_offset, }) }}
/// Keeps track of how much unused buffer space is left. The initial amount is the number of bytes/// requested by the user using the `buffers_size` field of `binder_transaction_data_sg`. Each time/// we translate an object of type `BINDER_TYPE_PTR`, some of the unused buffer space is consumed.struct UnusedBufferSpace { /// The start of the remaining space. offset: usize, /// The end of the remaining space. limit: usize,}impl UnusedBufferSpace { /// Claim the next `size` bytes from the unused buffer space. The offset for the claimed chunk /// into the buffer is returned. fn claim_next(&mut self, size: usize) -> Result<usize> { // We require every chunk to be aligned. let size = ptr_align(size).ok_or(EINVAL)?; let new_offset = self.offset.checked_add(size).ok_or(EINVAL)?;
if new_offset <= self.limit { let offset = self.offset; self.offset = new_offset; Ok(offset) } else { Err(EINVAL) } }}
#[must_use]pub(crate) enum PushWorkRes { Ok, OkNotifyPoll, FailedDead(DLArc<dyn DeliverToRead>),}
impl PushWorkRes { fn is_ok(&self) -> bool { match self { PushWorkRes::Ok => true, PushWorkRes::OkNotifyPoll => true, PushWorkRes::FailedDead(_) => false, } }}
/// The fields of `Thread` protected by the spinlock.struct InnerThread { /// Determines the looper state of the thread. It is a bit-wise combination of the constants /// prefixed with `LOOPER_`. looper_flags: LooperFlags,
/// Determines whether the looper should return. looper_need_return: bool,
/// Determines if thread is dead. is_dead: bool,
/// Work item used to deliver error codes to the thread that started a transaction. Stored here /// so that it can be reused. reply_work: DArc<ThreadError>,
/// Work item used to deliver error codes to the current thread. Stored here so that it can be /// reused. return_work: DArc<ThreadError>,
/// Determines whether the work list below should be processed. When set to false, `work_list` /// is treated as if it were empty. process_work_list: bool, /// List of work items to deliver to userspace. work_list: List<DTRWrap<dyn DeliverToRead>>, current_transaction: Option<DArc<Transaction>>,
/// Extended error information for this thread. extended_error: ExtendedError,}
kernel::impl_flags!( /// Represents multiple looper flags. #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] pub struct LooperFlags(u32);
/// Represents a single looper flag. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LooperFlag { Registered = bit_u32(0), Entered = bit_u32(1), Exited = bit_u32(2), Invalid = bit_u32(3), Waiting = bit_u32(4), WaitingProc = bit_u32(5), Poll = bit_u32(6), });
impl InnerThread { fn new(pid: i32) -> Result<Self> { fn next_err_id() -> u32 { static EE_ID: Atomic<u32> = Atomic::new(0); EE_ID.fetch_add(1, Relaxed) }
Ok(Self { looper_flags: LooperFlags::default(), looper_need_return: false, is_dead: false, process_work_list: false, reply_work: ThreadError::try_new(pid)?, return_work: ThreadError::try_new(pid)?, work_list: List::new(), current_transaction: None, extended_error: ExtendedError::new(next_err_id(), BR_OK, 0), }) }
fn pop_work(&mut self) -> Option<DLArc<dyn DeliverToRead>> { if !self.process_work_list { return None; }
let ret = self.work_list.pop_front(); self.process_work_list = !self.work_list.is_empty(); ret }
fn push_work(&mut self, work: DLArc<dyn DeliverToRead>) -> PushWorkRes { if self.is_dead { return PushWorkRes::FailedDead(work); } self.work_list.push_back(work); self.process_work_list = true; if self.looper_flags.contains(LooperFlag::Poll) { PushWorkRes::OkNotifyPoll } else { PushWorkRes::Ok } }
fn push_reply_work(&mut self, code: u32) -> PushWorkRes { if let Ok(work) = ListArc::try_from_arc(self.reply_work.clone()) { work.set_error_code(code); self.push_work(work) } else { pr_warn_ratelimited!("Thread reply work is already in use.\n"); PushWorkRes::Ok } }
fn push_return_work(&mut self, reply: u32) -> PushWorkRes { if let Ok(work) = ListArc::try_from_arc(self.return_work.clone()) { work.set_error_code(reply); self.push_work(work) } else { pr_warn_ratelimited!("Thread return work is already in use.\n"); PushWorkRes::Ok } }
/// Used to push work items that do not need to be processed immediately and can wait until the /// thread gets another work item. fn push_work_deferred(&mut self, work: DLArc<dyn DeliverToRead>) { self.work_list.push_back(work); }
/// Fetches the transaction this thread can reply to. If the thread has a pending transaction /// (that it could respond to) but it has also issued a transaction, it must first wait for the /// previously-issued transaction to complete. /// /// The `thread` parameter should be the thread containing this `ThreadInner`. fn pop_transaction_to_reply(&mut self, thread: &Thread) -> Result<DArc<Transaction>> { let transaction = self.current_transaction.take().ok_or(EINVAL)?; if core::ptr::eq(thread, transaction.from.as_ref()) { self.current_transaction = Some(transaction); return Err(EINVAL); } // Find a new current transaction for this thread. self.current_transaction = transaction.find_from(thread).cloned(); Ok(transaction) }
fn pop_transaction_replied(&mut self, transaction: &DArc<Transaction>) -> bool { match self.current_transaction.take() { None => false, Some(old) => { if !Arc::ptr_eq(transaction, &old) { self.current_transaction = Some(old); return false; } self.current_transaction = old.clone_next(); true } } }
fn looper_enter(&mut self) { self.looper_flags |= LooperFlag::Entered; if self.looper_flags.contains(LooperFlag::Registered) { self.looper_flags |= LooperFlag::Invalid; } }
fn looper_register(&mut self, valid: bool) { self.looper_flags |= LooperFlag::Registered; if !valid || self.looper_flags.contains(LooperFlag::Entered) { self.looper_flags |= LooperFlag::Invalid; } }
fn looper_exit(&mut self) { self.looper_flags |= LooperFlag::Exited; }
/// Determines whether the thread is part of a pool, i.e., if it is a looper. fn is_looper(&self) -> bool { self.looper_flags .contains_any(LooperFlag::Entered | LooperFlag::Registered) }
/// Determines whether the thread should attempt to fetch work items from the process queue. /// This is generally case when the thread is registered as a looper and not part of a /// transaction stack. But if there is local work, we want to return to userspace before we /// deliver any remote work. fn should_use_process_work_queue(&self) -> bool { self.current_transaction.is_none() && !self.process_work_list && self.is_looper() }
fn poll(&mut self) -> u32 { self.looper_flags |= LooperFlag::Poll; if self.process_work_list || self.looper_need_return { bindings::POLLIN } else { 0 } }}
/// This represents a thread that's used with binder.#[pin_data]pub(crate) struct Thread { pub(crate) id: i32, pub(crate) process: Arc<Process>, pub(crate) task: ARef<Task>, #[pin] inner: SpinLock<InnerThread>, #[pin] work_condvar: CondVar, /// Used to insert this thread into the process' `ready_threads` list. /// /// INVARIANT: May never be used for any other list than the `self.process.ready_threads`. #[pin] links: ListLinks, #[pin] links_track: AtomicTracker,}
kernel::list::impl_list_arc_safe! { impl ListArcSafe<0> for Thread { tracked_by links_track: AtomicTracker; }}kernel::list::impl_list_item! { impl ListItem<0> for Thread { using ListLinks { self.links }; }}
impl Thread { pub(crate) fn new(id: i32, process: Arc<Process>) -> Result<Arc<Self>> { let inner = InnerThread::new(process.task.pid())?;
Arc::pin_init( try_pin_init!(Thread { id, process, task: ARef::from(&**kernel::current!()), inner <- kernel::new_spinlock!(inner, "Thread::inner"), work_condvar <- kernel::new_condvar!("Thread::work_condvar"), links <- ListLinks::new(), links_track <- AtomicTracker::new(), }), GFP_KERNEL, ) }
#[inline(never)] pub(crate) fn debug_print(self: &Arc<Self>, m: &SeqFile, print_all: bool) -> Result { let inner = self.inner.lock();
if print_all || inner.current_transaction.is_some() || !inner.work_list.is_empty() { seq_print!( m, " thread {}: l {:02x} need_return {}\n", self.id, u32::from(inner.looper_flags), inner.looper_need_return, ); }
let mut t_opt = inner.current_transaction.as_ref(); while let Some(t) = t_opt { if Arc::ptr_eq(&t.from, self) { t.debug_print_inner(m, " outgoing transaction "); t_opt = t.from_parent.as_ref(); } else if Arc::ptr_eq(&t.to, &self.process) { t.debug_print_inner(m, " incoming transaction "); t_opt = t.find_from(self); } else { t.debug_print_inner(m, " bad transaction "); t_opt = None; } }
for work in &inner.work_list { work.debug_print(m, " ", " pending transaction ")?; } Ok(()) }
pub(crate) fn clear_extended_error(&self, debug_id: usize) { self.inner.lock().extended_error = ExtendedError::new(debug_id as u32, BR_OK, 0); }
pub(crate) fn get_extended_error(&self, data: UserSlice) -> Result { let mut writer = data.writer(); let mut inner = self.inner.lock(); let ee = inner.extended_error; inner.extended_error = ExtendedError::new(0, BR_OK, 0); drop(inner); writer.write(&ee)?; Ok(()) }
pub(crate) fn set_current_transaction(&self, transaction: DArc<Transaction>) { self.inner.lock().current_transaction = Some(transaction); }
pub(crate) fn has_current_transaction(&self) -> bool { self.inner.lock().current_transaction.is_some() }
/// Attempts to fetch a work item from the thread-local queue. The behaviour if the queue is /// empty depends on `wait`: if it is true, the function waits for some work to be queued (or a /// signal); otherwise it returns indicating that none is available. // #[export_name] is a temporary workaround so that ps output does not become unreadable from // mangled symbol names. #[export_name = "rust_binder_waitlcl"] fn get_work_local(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> { { let mut inner = self.inner.lock(); if inner.looper_need_return { return Ok(inner.pop_work()); } }
// Try once if the caller does not want to wait. if !wait { return self.inner.lock().pop_work().ok_or(EAGAIN).map(Some); }
// Loop waiting only on the local queue (i.e., not registering with the process queue). let mut inner = self.inner.lock(); loop { if let Some(work) = inner.pop_work() { return Ok(Some(work)); }
inner.looper_flags |= LooperFlag::Waiting; let signal_pending = self.work_condvar.wait_interruptible_freezable(&mut inner); inner.looper_flags &= !LooperFlag::Waiting;
if signal_pending { return Err(EINTR); } if inner.looper_need_return { return Ok(None); } } }
/// Attempts to fetch a work item from the thread-local queue, falling back to the process-wide /// queue if none is available locally. /// /// This must only be called when the thread is not participating in a transaction chain. If it /// is, the local version (`get_work_local`) should be used instead. // #[export_name] is a temporary workaround so that ps output does not become unreadable from // mangled symbol names. #[export_name = "rust_binder_wait"] fn get_work(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> { let thread_has_deferred_work;
// Try to get work from the thread's work queue, using only a local lock. { let mut inner = self.inner.lock();
// The process_work_list boolean is used to make us go to sleep even if there is work // in the thread todo-list, but it doesn't apply to the process todo-list. Furthermore, // work in the thread todo-list must still be delivered before the process list. // // Thus, in some scenarios we must return the thread work now even if we were requested // to wait. Adjust `process_work_list` to `true` accordingly. inner.process_work_list |= inner.looper_need_return; inner.process_work_list |= !wait;
if let Some(work) = inner.pop_work() { return Ok(Some(work)); } if inner.looper_need_return { drop(inner); return Ok(self.process.get_work()); }
// Note that if the thread list is empty, then the call to `pop_work()` has changed // `process_work_list` back to `false` even if we set it to `true` above. thread_has_deferred_work = !inner.work_list.is_empty(); }
// If the caller doesn't want to wait, try to grab work from the process queue. // // We know nothing will have been queued directly to the thread queue because it is not in // a transaction and it is not in the process' ready list. We also know the thread list has // no deferred work due to the `inner.process_work_list |= !wait` call above. if !wait { return self.process.get_work().ok_or(EAGAIN).map(Some); }
// Get work from the process queue. If none is available, atomically register as ready. let reg = match self .process .get_work_or_register(self, thread_has_deferred_work) { GetWorkOrRegister::Work(work) => return Ok(Some(work)), GetWorkOrRegister::Register(reg) => reg, };
let mut inner = self.inner.lock(); loop { if let Some(work) = inner.pop_work() { return Ok(Some(work)); }
inner.looper_flags |= LooperFlag::Waiting | LooperFlag::WaitingProc; let signal_pending = self.work_condvar.wait_interruptible_freezable(&mut inner); inner.looper_flags &= !(LooperFlag::Waiting | LooperFlag::WaitingProc);
if signal_pending || inner.looper_need_return { // We need to return now. // // We need to pull the thread off the list of ready threads (by dropping `reg`), // then check the state again after it's off the list to ensure that something was // not queued in the meantime. If something has been queued (or if there is // deferred work), we just return it (instead of the error). drop(inner); drop(reg);
inner = self.inner.lock(); inner.process_work_list = true; let res = match inner.pop_work() { Some(work) => Ok(Some(work)), None if signal_pending => Err(EINTR), None => Ok(None), }; return res; } } }
/// Push the provided work item to be delivered to user space via this thread. /// /// Returns whether the item was successfully pushed. This can only fail if the thread is dead. pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> PushWorkRes { let sync = work.should_sync_wakeup(); self.push_work_inner(work, sync) }
pub(crate) fn push_work_inner( &self, work: DLArc<dyn DeliverToRead>, sync: bool, ) -> PushWorkRes { let res = self.inner.lock().push_work(work);
if res.is_ok() { if sync { self.work_condvar.notify_sync(); } else { self.work_condvar.notify_one(); } }
res }
/// Attempts to push to given work item to the thread if it's a looper thread (i.e., if it's /// part of a thread pool) and is alive. Otherwise, push the work item to the process instead. pub(crate) fn push_work_if_looper(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult { let mut inner = self.inner.lock(); if inner.is_looper() && !inner.is_dead { // Not notifying work_condvar: Reply to current thread. let res = inner.push_work(work); drop(inner); if matches!(res, PushWorkRes::OkNotifyPoll) { self.process.notify_poll(false); } Ok(()) } else { drop(inner); self.process.push_work(work) } }
pub(crate) fn push_work_deferred(&self, work: DLArc<dyn DeliverToRead>) { self.inner.lock().push_work_deferred(work); }
pub(crate) fn push_return_work(&self, reply: u32) { // Not notifying work_condvar: Reply to current thread. let res = self.inner.lock().push_return_work(reply); if matches!(res, PushWorkRes::OkNotifyPoll) { self.process.notify_poll(false); } }
pub(crate) fn pop_work_even_if_deferred(&self) -> Option<DLArc<dyn DeliverToRead>> { let mut thread_inner = self.inner.lock(); thread_inner.process_work_list = true; thread_inner.pop_work() }
fn translate_object( &self, obj_index: usize, offset: usize, object: BinderObjectRef<'_>, view: &mut AllocationView<'_>, allow_fds: bool, sg_state: &mut ScatterGatherState, ) -> BinderResult { match object { BinderObjectRef::Binder(obj) => { let strong = obj.hdr.type_ == BINDER_TYPE_BINDER; // SAFETY: `binder` is a `binder_uintptr_t`; any bit pattern is a valid // representation. let ptr = unsafe { obj.__bindgen_anon_1.binder }; let cookie = obj.cookie; let flags = obj.flags; let node = self .process .as_arc_borrow() .get_node(ptr, cookie, flags, strong, self)?; security::binder_transfer_binder(&self.process.cred, &view.alloc.process.cred)?; view.transfer_binder_object(offset, obj, strong, node)?; } BinderObjectRef::Handle(obj) => { let strong = obj.hdr.type_ == BINDER_TYPE_HANDLE; // SAFETY: `handle` is a `u32`; any bit pattern is a valid representation. let handle = unsafe { obj.__bindgen_anon_1.handle }; let node = self.process.get_node_from_handle(handle, strong)?; security::binder_transfer_binder(&self.process.cred, &view.alloc.process.cred)?; view.transfer_binder_object(offset, obj, strong, node)?; } BinderObjectRef::Fd(obj) => { if !allow_fds { return Err(EPERM.into()); }
// SAFETY: `fd` is a `u32`; any bit pattern is a valid representation. let fd = unsafe { obj.__bindgen_anon_1.fd }; let file = LocalFile::fget(fd)?; // SAFETY: The binder driver never calls `fdget_pos` and this code runs from an // ioctl, so there are no active calls to `fdget_pos` on this thread. let file = unsafe { LocalFile::assume_no_fdget_pos(file) }; security::binder_transfer_file( &self.process.cred, &view.alloc.process.cred, &file, )?;
let mut obj_write = BinderFdObject::default(); obj_write.hdr.type_ = BINDER_TYPE_FD; // This will be overwritten with the actual fd when the transaction is received. obj_write.__bindgen_anon_1.fd = u32::MAX; obj_write.cookie = obj.cookie; view.write::<BinderFdObject>(offset, &obj_write)?;
const FD_FIELD_OFFSET: usize = core::mem::offset_of!(uapi::binder_fd_object, __bindgen_anon_1.fd);
let field_offset = offset + FD_FIELD_OFFSET; crate::trace::trace_transaction_fd_send(view.alloc.debug_id, fd, field_offset);
view.alloc.info_add_fd(file, field_offset, false)?; } BinderObjectRef::Ptr(obj) => { let obj_length = obj.length.try_into().map_err(|_| EINVAL)?; let alloc_offset = match sg_state.unused_buffer_space.claim_next(obj_length) { Ok(alloc_offset) => alloc_offset, Err(err) => { binder_debug!( UserError, "failed to claim space for a BINDER_TYPE_PTR (offset: {}, limit: {}, size: {})", sg_state.unused_buffer_space.offset, sg_state.unused_buffer_space.limit, obj_length ); return Err(err.into()); } };
let sg_state_idx = sg_state.sg_entries.len(); sg_state.sg_entries.push( ScatterGatherEntry { obj_index, offset: alloc_offset, sender_uaddr: obj.buffer as usize, length: obj_length, pointer_fixups: KVec::new(), fixup_min_offset: 0, }, GFP_KERNEL, )?;
let buffer_ptr_in_user_space = (view.alloc.ptr + alloc_offset) as u64;
if obj.flags & uapi::BINDER_BUFFER_FLAG_HAS_PARENT == 0 { sg_state.ancestors.clear(); sg_state.ancestors.push(sg_state_idx, GFP_KERNEL)?; } else { // Another buffer also has a pointer to this buffer, and we need to fixup that // pointer too.
let parent_index = usize::try_from(obj.parent).map_err(|_| EINVAL)?; let parent_offset = usize::try_from(obj.parent_offset).map_err(|_| EINVAL)?;
let info = sg_state.validate_parent_fixup( parent_index, parent_offset, size_of::<u64>(), )?;
sg_state.ancestors.truncate(info.num_ancestors); sg_state.ancestors.push(sg_state_idx, GFP_KERNEL)?;
let parent_entry = match sg_state.sg_entries.get_mut(info.parent_sg_index) { Some(parent_entry) => parent_entry, None => { pr_err_ratelimited!( "validate_parent_fixup returned index out of bounds for sg.entries\n" ); return Err(EINVAL.into()); } };
parent_entry.fixup_min_offset = info.new_min_offset; parent_entry.pointer_fixups.push( PointerFixupEntry::Fixup { pointer_value: buffer_ptr_in_user_space, target_offset: info.target_offset, }, GFP_KERNEL, )?; }
let mut obj_write = BinderBufferObject::default(); obj_write.hdr.type_ = BINDER_TYPE_PTR; obj_write.flags = obj.flags; obj_write.buffer = buffer_ptr_in_user_space; obj_write.length = obj.length; obj_write.parent = obj.parent; obj_write.parent_offset = obj.parent_offset; view.write::<BinderBufferObject>(offset, &obj_write)?; } BinderObjectRef::Fda(obj) => { if !allow_fds { return Err(EPERM.into()); } let parent_index = usize::try_from(obj.parent).map_err(|_| EINVAL)?; let parent_offset = usize::try_from(obj.parent_offset).map_err(|_| EINVAL)?; let num_fds = usize::try_from(obj.num_fds).map_err(|_| EINVAL)?; let fds_len = num_fds.checked_mul(size_of::<u32>()).ok_or(EINVAL)?;
if !is_aligned(parent_offset, size_of::<u32>()) { binder_debug!(UserError, "FDA parent offset not aligned correctly"); return Err(EINVAL.into()); }
let info = sg_state.validate_parent_fixup(parent_index, parent_offset, fds_len)?; view.alloc.info_add_fd_reserve(num_fds)?;
sg_state.ancestors.truncate(info.num_ancestors); let parent_entry = match sg_state.sg_entries.get_mut(info.parent_sg_index) { Some(parent_entry) => parent_entry, None => { pr_err_ratelimited!( "validate_parent_fixup returned index out of bounds for sg.entries\n" ); return Err(EINVAL.into()); } };
if !is_aligned(parent_entry.sender_uaddr, size_of::<u32>()) { binder_debug!(UserError, "FDA parent buffer not aligned correctly"); return Err(EINVAL.into()); }
parent_entry.fixup_min_offset = info.new_min_offset; parent_entry .pointer_fixups .push( PointerFixupEntry::Skip { skip: fds_len, target_offset: info.target_offset, }, GFP_KERNEL, ) .map_err(|_| ENOMEM)?;
let fda_uaddr = parent_entry .sender_uaddr .checked_add(parent_offset) .ok_or(EINVAL)?;
let mut fda_bytes = KVec::new(); UserSlice::new(UserPtr::from_addr(fda_uaddr as usize), fds_len) .read_all(&mut fda_bytes, GFP_KERNEL)?;
if fds_len != fda_bytes.len() { pr_err_ratelimited!( "UserSlice::read_all returned wrong length in BINDER_TYPE_FDA\n" ); return Err(EINVAL.into()); }
for i in (0..fds_len).step_by(size_of::<u32>()) { let fd = { let mut fd_bytes = [0u8; size_of::<u32>()]; fd_bytes.copy_from_slice(&fda_bytes[i..i + size_of::<u32>()]); u32::from_ne_bytes(fd_bytes) };
let file = LocalFile::fget(fd)?; // SAFETY: The binder driver never calls `fdget_pos` and this code runs from an // ioctl, so there are no active calls to `fdget_pos` on this thread. let file = unsafe { LocalFile::assume_no_fdget_pos(file) }; security::binder_transfer_file( &self.process.cred, &view.alloc.process.cred, &file, )?;
// The `validate_parent_fixup` call ensuers that this addition will not // overflow. view.alloc.info_add_fd(file, info.target_offset + i, true)?; } drop(fda_bytes);
let mut obj_write = BinderFdArrayObject::default(); obj_write.hdr.type_ = BINDER_TYPE_FDA; obj_write.num_fds = obj.num_fds; obj_write.parent = obj.parent; obj_write.parent_offset = obj.parent_offset; view.write::<BinderFdArrayObject>(offset, &obj_write)?; } } Ok(()) }
fn apply_sg(&self, alloc: &mut Allocation, sg_state: &mut ScatterGatherState) -> BinderResult { for sg_entry in &mut sg_state.sg_entries { let mut end_of_previous_fixup = sg_entry.offset; let offset_end = sg_entry.offset.checked_add(sg_entry.length).ok_or(EINVAL)?;
let mut reader = UserSlice::new(UserPtr::from_addr(sg_entry.sender_uaddr), sg_entry.length).reader(); for fixup in &mut sg_entry.pointer_fixups { let (fixup_len, fixup_offset) = match fixup { PointerFixupEntry::Fixup { target_offset, .. } => { (size_of::<u64>(), *target_offset) } PointerFixupEntry::Skip { skip, target_offset, } => (*skip, *target_offset), };
let target_offset_end = fixup_offset.checked_add(fixup_len).ok_or(EINVAL)?; if fixup_offset < end_of_previous_fixup || offset_end < target_offset_end { binder_debug!( UserError, "fixups oob {fixup_offset} {end_of_previous_fixup} {offset_end} {target_offset_end}" ); return Err(EINVAL.into()); }
let copy_off = end_of_previous_fixup; let copy_len = fixup_offset - end_of_previous_fixup; if let Err(err) = alloc.copy_into(&mut reader, copy_off, copy_len) { binder_debug!(UserError, "failed copying into alloc: {err:?}"); return Err(err.into()); } if let PointerFixupEntry::Fixup { pointer_value, .. } = fixup { let res = alloc.write::<u64>(fixup_offset, pointer_value); if let Err(err) = res { binder_debug!(UserError, "failed copying ptr into alloc: {err:?}"); return Err(err.into()); } } if let Err(err) = reader.skip(fixup_len) { binder_debug!( UserError, "failed skipping {fixup_len} from reader: {err:?}" ); return Err(err.into()); } end_of_previous_fixup = target_offset_end; } let copy_off = end_of_previous_fixup; let copy_len = offset_end - end_of_previous_fixup; if let Err(err) = alloc.copy_into(&mut reader, copy_off, copy_len) { binder_debug!(UserError, "failed copying remainder into alloc: {err:?}"); return Err(err.into()); } } Ok(()) }
/// This method copies the payload of a transaction into the target process. /// /// The resulting payload will have several different components, which will be stored next to /// each other in the allocation. Furthermore, various objects can be embedded in the payload, /// and those objects have to be translated so that they make sense to the target transaction. pub(crate) fn copy_transaction_data( &self, to_process: Arc<Process>, info: &mut TransactionInfo, debug_id: usize, allow_fds: bool, txn_security_ctx_offset: Option<&mut usize>, ) -> BinderResult<NewAllocation> { let mut secctx = if let Some(offset) = txn_security_ctx_offset { let secid = self.process.cred.get_secid(); let ctx = match security::SecurityCtx::from_secid(secid) { Ok(ctx) => ctx, Err(err) => { pr_warn_ratelimited!( "Failed to get security ctx for id {}: {:?}\n", secid, err ); return Err(err.into()); } }; Some((offset, ctx)) } else { None };
let data_size = info.data_size; let aligned_data_size = ptr_align(data_size).ok_or(EINVAL)?; let offsets_size = info.offsets_size; let buffers_size = info.buffers_size; let aligned_secctx_size = match secctx.as_ref() { Some((_offset, ctx)) => ptr_align(ctx.len()).ok_or(EINVAL)?, None => 0, };
if !is_aligned(offsets_size, size_of::<u64>()) { return Err(EINVAL.into()); } if !is_aligned(buffers_size, size_of::<u64>()) { return Err(EINVAL.into()); }
// This guarantees that at least `sizeof(usize)` bytes will be allocated. let len = usize::max( aligned_data_size .checked_add(offsets_size) .and_then(|sum| sum.checked_add(buffers_size)) .and_then(|sum| sum.checked_add(aligned_secctx_size)) .ok_or(ENOMEM)?, size_of::<u64>(), ); let secctx_off = aligned_data_size + offsets_size + buffers_size; let mut alloc = to_process.buffer_alloc(debug_id, len, info)?;
trace_transaction_alloc_buf(debug_id, data_size, offsets_size, buffers_size);
let mut buffer_reader = UserSlice::new(info.data_ptr, data_size).reader(); let mut end_of_previous_object = 0; let mut sg_state = None;
// Copy offsets if there are any. if offsets_size > 0 { let mut offsets_reader = UserSlice::new(info.offsets_ptr, offsets_size).reader();
let offsets_start = aligned_data_size; let offsets_end = aligned_data_size + offsets_size;
// This state is used for BINDER_TYPE_PTR objects. let sg_state = sg_state.insert(ScatterGatherState { unused_buffer_space: UnusedBufferSpace { offset: offsets_end, limit: offsets_end + buffers_size, }, sg_entries: KVec::new(), ancestors: KVec::new(), });
// Traverse the objects specified. let mut view = AllocationView::new(&mut alloc, data_size); for (index, index_offset) in (offsets_start..offsets_end) .step_by(size_of::<u64>()) .enumerate() { let offset = offsets_reader.read::<u64>()?; view.alloc.write(index_offset, &offset)?; let offset: usize = offset.try_into().map_err(|_| EINVAL)?;
if offset < end_of_previous_object || !is_aligned(offset, size_of::<u32>()) { binder_debug!(UserError, "got transaction with invalid offset"); return Err(EINVAL.into()); }
// Copy data between two objects. if end_of_previous_object < offset { view.copy_into( &mut buffer_reader, end_of_previous_object, offset - end_of_previous_object, )?; }
let mut object = BinderObject::read_from(&mut buffer_reader)?;
match self.translate_object( index, offset, object.as_ref(), &mut view, allow_fds, sg_state, ) { Ok(()) => end_of_previous_object = offset + object.size(), Err(err) => { binder_debug!(UserError, "error while translating object: {err:?}"); return Err(err); } }
// Update the indexes containing objects to clean up. let offset_after_object = index_offset + size_of::<u64>(); view.alloc .set_info_offsets(offsets_start..offset_after_object); } }
// Copy remaining raw data. alloc.copy_into( &mut buffer_reader, end_of_previous_object, data_size - end_of_previous_object, )?;
if let Some(sg_state) = sg_state.as_mut() { self.apply_sg(&mut alloc, sg_state)?; }
if let Some((off_out, secctx)) = secctx.as_mut() { if let Err(err) = alloc.write(secctx_off, secctx.as_bytes()) { binder_debug!(UserError, "failed to write security context: {err:?}"); return Err(err.into()); } **off_out = secctx_off; } Ok(alloc) }
fn unwind_transaction_stack(self: &Arc<Self>) { let mut thread = self.clone(); while let Ok(transaction) = { let mut inner = thread.inner.lock(); inner.pop_transaction_to_reply(thread.as_ref()) } { binder_debug!( DeadTransaction, "release transaction {} in, still active", transaction.debug_id );
let reply = Err(BR_DEAD_REPLY); if !transaction .from .deliver_single_reply(reply, &transaction, None) { break; }
thread = transaction.from.clone(); } }
pub(crate) fn deliver_reply( &self, reply: Result<DLArc<Transaction>, u32>, transaction: &DArc<Transaction>, extended_error: Option<ExtendedError>, ) { if self.deliver_single_reply(reply, transaction, extended_error) { transaction.from.unwind_transaction_stack(); } }
/// Delivers a reply to the thread that started a transaction. The reply can either be a /// reply-transaction or an error code to be delivered instead. /// /// Returns whether the thread is dead. If it is, the caller is expected to unwind the /// transaction stack by completing transactions for threads that are dead. fn deliver_single_reply( &self, reply: Result<DLArc<Transaction>, u32>, transaction: &DArc<Transaction>, extended_error: Option<ExtendedError>, ) -> bool { if let Ok(transaction) = &reply { crate::trace::trace_transaction(true, transaction, Some(&self.task)); transaction.set_outstanding(&mut self.process.inner.lock()); }
let ret = { let mut inner = self.inner.lock(); if !inner.pop_transaction_replied(transaction) { return false; }
if inner.is_dead { return true; }
if let Some(ee) = extended_error { if inner.extended_error.command == BR_OK { inner.extended_error = ee; } }
match reply { Ok(work) => inner.push_work(work), Err(code) => inner.push_reply_work(code), } };
// Notify the thread now that we've released the inner lock. self.work_condvar.notify_sync(); if matches!(ret, PushWorkRes::OkNotifyPoll) { self.process.notify_poll(true); } false }
/// Determines if the given transaction is the current transaction for this thread. fn is_current_transaction(&self, transaction: &DArc<Transaction>) -> bool { let inner = self.inner.lock(); match &inner.current_transaction { None => false, Some(current) => Arc::ptr_eq(current, transaction), } }
/// Determines the current top of the transaction stack. It fails if the top is in another /// thread (i.e., this thread belongs to a stack but it has called another thread). The top is /// [`None`] if the thread is not currently participating in a transaction stack. fn top_of_transaction_stack(&self) -> Result<Option<DArc<Transaction>>> { let inner = self.inner.lock(); if let Some(cur) = &inner.current_transaction { if core::ptr::eq(self, cur.from.as_ref()) { pr_warn_ratelimited!("got new transaction with bad transaction stack\n"); return Err(EINVAL); } Ok(Some(cur.clone())) } else { Ok(None) } }
// No inlining avoids allocating stack space for `BinderTransactionData` for the entire // duration of `transaction()`. #[inline(never)] fn read_transaction_info( &self, cmd: u32, reader: &mut UserSliceReader, info: &mut TransactionInfo, ) -> Result { let td = match cmd { BC_TRANSACTION | BC_REPLY => { reader.read::<BinderTransactionData>()?.with_buffers_size(0) } BC_TRANSACTION_SG | BC_REPLY_SG => reader.read::<BinderTransactionDataSg>()?, _ => return Err(EINVAL), };
// SAFETY: Above `read` call initializes all bytes, so this union read is ok. let trd_data_ptr = unsafe { &td.transaction_data.data.ptr };
info.is_reply = matches!(cmd, BC_REPLY | BC_REPLY_SG); info.from_pid = self.process.task.pid(); info.from_tid = self.id; info.code = td.transaction_data.code; info.flags = TransactionFlags::from_bits(td.transaction_data.flags); info.data_ptr = UserPtr::from_addr(trd_data_ptr.buffer as usize); info.data_size = td.transaction_data.data_size as usize; info.offsets_ptr = UserPtr::from_addr(trd_data_ptr.offsets as usize); info.offsets_size = td.transaction_data.offsets_size as usize; info.buffers_size = td.buffers_size as usize; // SAFETY: Above `read` call initializes all bytes, so this union read is ok. info.target_handle = unsafe { td.transaction_data.target.handle };
info.debug_id = super::next_debug_id();
Ok(()) }
#[inline(never)] fn transaction(self: &Arc<Self>, cmd: u32, reader: &mut UserSliceReader) -> Result { let mut info = TransactionInfo::zeroed(); self.read_transaction_info(cmd, reader, &mut info)?;
self.clear_extended_error(info.debug_id);
let ret = if info.is_reply { self.reply_inner(&mut info) } else if info.is_oneway() { self.oneway_transaction_inner(&mut info) } else { self.transaction_inner(&mut info) };
// This runs when return work is passed to the caller. This is not // always the same as the transaction failing, as reply errors are // delivered to the remote process. if let Err(err) = ret { self.push_return_work(err.reply); if err.reply != BR_TRANSACTION_COMPLETE { info.reply = err.reply; info.error_line = Some(err.line); if let Some(source) = err.source { info.errno = Some(source);
self.inner.lock().extended_error = ExtendedError::new(info.debug_id as u32, err.reply, source.to_errno()); } } }
info.write_log(&self.process.ctx);
if info.oneway_spam_suspect { // If this is both a oneway spam suspect and a failure, we report it twice. This is // useful in case the transaction failed with BR_TRANSACTION_PENDING_FROZEN. info.report_netlink(BR_ONEWAY_SPAM_SUSPECT, &self.process.ctx); } // This runs when the transaction failed. if info.reply != 0 { info.report_netlink(info.reply, &self.process.ctx); if let Some(errno) = info.errno { let error_line: &dyn kernel::fmt::Display = match &info.error_line { Some(line) => line, None => &0, }; binder_debug!( FailedTransaction, "transaction {} to {}:{} failed {:?}, code {} size {}-{} line {}", if info.is_reply { "reply" } else if info.is_oneway() { "async" } else { "call" }, info.to_pid, info.to_tid, errno, info.code, info.data_size, info.offsets_size, error_line ); } }
Ok(()) }
fn transaction_inner(self: &Arc<Self>, info: &mut TransactionInfo) -> BinderResult { let node_ref = self.process.get_transaction_node(info.target_handle)?; info.to_pid = node_ref.node.owner.task.pid(); info.to_node_debug_id = node_ref.node.debug_id; security::binder_transaction(&self.process.cred, &node_ref.node.owner.cred)?; // TODO: We need to ensure that there isn't a pending transaction in the work queue. How // could this happen? let top = self.top_of_transaction_stack()?; let list_completion = DTRWrap::arc_try_new(DeliverCode::new( BR_TRANSACTION_COMPLETE, self.process.task.pid(), ))?; let completion = list_completion.clone_arc(); let transaction = Transaction::new(node_ref, top, self, info)?;
// Check that the transaction stack hasn't changed while the lock was released, then update // it with the new transaction. { let mut inner = self.inner.lock(); if !transaction.is_stacked_on(&inner.current_transaction) { binder_debug!(UserError, "got new transaction with bad transaction stack"); return Err(EINVAL.into()); } inner.current_transaction = Some(transaction.clone_arc()); // We push the completion as a deferred work so that we wait for the reply before // returning to userland. inner.push_work_deferred(list_completion); }
if let Err(e) = transaction.submit(info) { completion.skip(); // Define `transaction` first to drop it after `inner`. let transaction; let mut inner = self.inner.lock(); transaction = inner.current_transaction.take().unwrap(); inner.current_transaction = transaction.clone_next(); Err(e) } else { Ok(()) } }
fn reply_inner(self: &Arc<Self>, info: &mut TransactionInfo) -> BinderResult { let orig = match self.inner.lock().pop_transaction_to_reply(self) { Ok(orig) => orig, Err(err) => { binder_debug!(UserError, "got reply transaction with no transaction stack"); return Err(err.into()); } }; if !orig.from.is_current_transaction(&orig) { binder_debug!( UserError, "got reply transaction with bad transaction stack" ); return Err(EINVAL.into()); }
info.to_tid = orig.from.id; info.to_pid = orig.from.process.task.pid();
// We need to complete the transaction even if we cannot complete building the reply. let out = (|| -> BinderResult<_> { let completion = DTRWrap::arc_try_new(DeliverCode::new( BR_TRANSACTION_COMPLETE, self.process.task.pid(), ))?; let process = orig.from.process.clone(); let allow_fds = orig.flags.contains(TransactionFlag::AcceptFds); let reply = Transaction::new_reply(self, process, info, allow_fds)?; if info.flags.contains(TransactionFlag::DeferComplete) { // The flag is set. Perform a deferred push so that `read` can wait for the // next incoming transaction without a userspace roundtrip. self.inner.lock().push_work_deferred(completion); } else { // Not notifying work_condvar: Reply to current thread. let res = self.inner.lock().push_work(completion); if matches!(res, PushWorkRes::OkNotifyPoll) { self.process.notify_poll(false); } } orig.from.deliver_reply(Ok(reply), &orig, None); Ok(()) })() .map_err(|mut err| { // At this point we only return `BR_TRANSACTION_COMPLETE` to the caller, and we must let // the sender know that the transaction has completed (with an error in this case).
let param = err.source.as_ref().map_or(0, |e| e.to_errno()); let ee = ExtendedError::new(info.debug_id as u32, err.reply, param); orig.from .deliver_reply(Err(BR_FAILED_REPLY), &orig, Some(ee)); info.reply = BR_FAILED_REPLY; info.errno = err.source; info.error_line = Some(err.line); err.reply = BR_TRANSACTION_COMPLETE; err });
out }
fn oneway_transaction_inner(self: &Arc<Self>, info: &mut TransactionInfo) -> BinderResult { let node_ref = self.process.get_transaction_node(info.target_handle)?; info.to_pid = node_ref.node.owner.task.pid(); info.to_node_debug_id = node_ref.node.debug_id; security::binder_transaction(&self.process.cred, &node_ref.node.owner.cred)?; let transaction = Transaction::new(node_ref, None, self, info)?; let code = if self.process.is_oneway_spam_detection_enabled() && info.oneway_spam_suspect { BR_ONEWAY_SPAM_SUSPECT } else { BR_TRANSACTION_COMPLETE }; let list_completion = DTRWrap::arc_try_new(DeliverCode::new(code, self.process.task.pid()))?; let completion = list_completion.clone_arc(); // Not notifying work_condvar: Reply to current thread. let res = self.inner.lock().push_work(list_completion); match transaction.submit(info) { Ok(()) => { if matches!(res, PushWorkRes::OkNotifyPoll) { self.process.notify_poll(false); } Ok(()) } Err(err) => { completion.skip(); Err(err) } } }
fn write(self: &Arc<Self>, req: &mut BinderWriteRead) -> Result { let write_start = req.write_buffer.wrapping_add(req.write_consumed); let write_len = req.write_size.saturating_sub(req.write_consumed); let mut reader = UserSlice::new(UserPtr::from_addr(write_start as usize), write_len as usize).reader();
while reader.len() >= size_of::<u32>() && self.inner.lock().return_work.is_unused() { let before = reader.len(); let cmd = reader.read::<u32>()?; crate::trace::trace_command(cmd); GLOBAL_STATS.inc_bc(cmd); self.process.stats.inc_bc(cmd); match cmd { BC_TRANSACTION | BC_TRANSACTION_SG | BC_REPLY | BC_REPLY_SG => { self.transaction(cmd, &mut reader)?; } BC_FREE_BUFFER => { let buffer = self.process.buffer_get(reader.read()?); if let Some(buffer) = buffer { if buffer.looper_need_return_on_free() { self.inner.lock().looper_need_return = true; } trace_transaction_buffer_release(buffer.debug_id); drop(buffer); } } BC_INCREFS => { self.process .as_arc_borrow() .update_ref(reader.read()?, true, false)? } BC_ACQUIRE => { self.process .as_arc_borrow() .update_ref(reader.read()?, true, true)? } BC_RELEASE => { self.process .as_arc_borrow() .update_ref(reader.read()?, false, true)? } BC_DECREFS => { self.process .as_arc_borrow() .update_ref(reader.read()?, false, false)? } BC_INCREFS_DONE => self.process.inc_ref_done(&mut reader, false)?, BC_ACQUIRE_DONE => self.process.inc_ref_done(&mut reader, true)?, BC_REQUEST_DEATH_NOTIFICATION => self.process.request_death(&mut reader, self)?, BC_CLEAR_DEATH_NOTIFICATION => self.process.clear_death(&mut reader, self)?, BC_DEAD_BINDER_DONE => self.process.dead_binder_done(reader.read()?, self), BC_REGISTER_LOOPER => { let valid = self.process.register_thread(); self.inner.lock().looper_register(valid); } BC_ENTER_LOOPER => self.inner.lock().looper_enter(), BC_EXIT_LOOPER => self.inner.lock().looper_exit(), BC_REQUEST_FREEZE_NOTIFICATION => self.process.request_freeze_notif(&mut reader)?, BC_CLEAR_FREEZE_NOTIFICATION => self.process.clear_freeze_notif(&mut reader)?, BC_FREEZE_NOTIFICATION_DONE => self.process.freeze_notif_done(&mut reader)?,
// Fail if given an unknown error code. // BC_ATTEMPT_ACQUIRE and BC_ACQUIRE_RESULT are no longer supported. _ => return Err(EINVAL), } // Update the number of write bytes consumed. req.write_consumed += (before - reader.len()) as u64; }
Ok(()) }
fn read(self: &Arc<Self>, req: &mut BinderWriteRead, wait: bool) -> Result { let read_start = req.read_buffer.wrapping_add(req.read_consumed); let read_len = req.read_size.saturating_sub(req.read_consumed); let mut writer = BinderReturnWriter::new( UserSlice::new(UserPtr::from_addr(read_start as usize), read_len as usize).writer(), self, ); let (in_pool, has_transaction, thread_todo, use_proc_queue) = { let inner = self.inner.lock(); ( inner.is_looper(), inner.current_transaction.is_some(), !inner.work_list.is_empty(), inner.should_use_process_work_queue(), ) };
crate::trace::trace_wait_for_work(use_proc_queue, has_transaction, thread_todo);
let getter = if use_proc_queue { Self::get_work } else { Self::get_work_local };
// Reserve some room at the beginning of the read buffer so that we can send a // BR_SPAWN_LOOPER if we need to. let mut has_noop_placeholder = false; if req.read_consumed == 0 { if let Err(err) = writer.write_code(BR_NOOP) { pr_warn_ratelimited!("Failure when writing BR_NOOP at beginning of buffer.\n"); return Err(err); } has_noop_placeholder = true; }
// Loop doing work while there is room in the buffer. let initial_len = writer.len(); while writer.len() >= size_of::<uapi::binder_transaction_data_secctx>() + 4 { match getter(self, wait && initial_len == writer.len()) { Ok(Some(work)) => match work.into_arc().do_work(self, &mut writer) { Ok(true) => {} Ok(false) => break, Err(err) => { return Err(err); } }, Ok(None) => { break; } Err(err) => { // Propagate the error if we haven't written anything else. if err != EINTR && err != EAGAIN { pr_warn_ratelimited!("Failure in work getter: {:?}\n", err); } if initial_len == writer.len() { return Err(err); } else { break; } } } }
req.read_consumed += read_len - writer.len() as u64;
// Write BR_SPAWN_LOOPER if the process needs more threads for its pool. if has_noop_placeholder && in_pool && self.process.needs_thread() { let mut writer = UserSlice::new( UserPtr::from_addr(req.read_buffer as usize), req.read_size as usize, ) .writer(); writer.write(&BR_SPAWN_LOOPER)?; } Ok(()) }
pub(crate) fn write_read(self: &Arc<Self>, data: UserSlice, wait: bool) -> Result { let (mut reader, mut writer) = data.reader_writer(); let mut req = reader.read::<BinderWriteRead>()?;
// Go through the write buffer. let mut ret = Ok(()); if req.write_size > 0 { ret = self.write(&mut req); crate::trace::trace_write_done(ret); if let Err(err) = ret { pr_warn_ratelimited!( "Write failure {:?} in pid:{}\n", err, self.process.pid_in_current_ns() ); req.read_consumed = 0; writer.write(&req)?; self.inner.lock().looper_need_return = false; return ret; } }
// Go through the work queue. if req.read_size > 0 { ret = self.read(&mut req, wait); crate::trace::trace_read_done(ret); if ret.is_err() && ret != Err(EINTR) { pr_warn_ratelimited!( "Read failure {:?} in pid:{}\n", ret, self.process.pid_in_current_ns() ); } }
// Write the request back so that the consumed fields are visible to the caller. writer.write(&req)?;
self.inner.lock().looper_need_return = false;
ret }
pub(crate) fn poll(&self) -> Result<(bool, u32)> { let mut inner = self.inner.lock(); Ok((inner.should_use_process_work_queue(), inner.poll())) }
/// Make the call to `get_work` or `get_work_local` return immediately, if any. pub(crate) fn exit_looper(&self) { let mut inner = self.inner.lock(); let should_notify = inner.looper_flags.contains(LooperFlag::Waiting); if should_notify { inner.looper_need_return = true; } drop(inner);
if should_notify { self.work_condvar.notify_one(); } }
pub(crate) fn release(self: &Arc<Self>) { self.inner.lock().is_dead = true;
self.unwind_transaction_stack();
// Cancel all pending work items. while let Some(work) = self.pop_work_even_if_deferred() { work.into_arc().cancel(); } }}
#[pin_data]struct ThreadError { error_code: Atomic<u32>, pid: i32, #[pin] links_track: AtomicTracker,}
impl ThreadError { fn try_new(pid: i32) -> Result<DArc<Self>> { DTRWrap::arc_pin_init(pin_init!(Self { error_code: Atomic::new(BR_OK), pid, links_track <- AtomicTracker::new(), })) .map(ListArc::into_arc) }
fn set_error_code(&self, code: u32) { self.error_code.store(code, Relaxed); }
fn is_unused(&self) -> bool { self.error_code.load(Relaxed) == BR_OK }}
impl DeliverToRead for ThreadError { fn do_work( self: DArc<Self>, _thread: &Thread, writer: &mut BinderReturnWriter<'_>, ) -> Result<bool> { let code = self.error_code.load(Relaxed); self.error_code.store(BR_OK, Relaxed); writer.write_code(code)?; Ok(true) }
fn cancel(self: DArc<Self>) { let code = self.error_code.load(Relaxed); if code != BR_OK { binder_debug!( pid = self.pid, DeadTransaction, "undelivered TRANSACTION_ERROR: {code}" ); } }
fn should_sync_wakeup(&self) -> bool { false }
fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result { seq_print!( m, "{}transaction error: {}\n", prefix, self.error_code.load(Relaxed) ); Ok(()) }}
kernel::list::impl_list_arc_safe! { impl ListArcSafe<0> for ThreadError { tracked_by links_track: AtomicTracker; }}