Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293// SPDX-License-Identifier: GPL-2.0-or-later/* * Queued read/write locks * * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. * * Authors: Waiman Long <waiman.long@hp.com> */#include <linux/smp.h>#include <linux/bug.h>#include <linux/cpumask.h>#include <linux/percpu.h>#include <linux/hardirq.h>#include <linux/spinlock.h>#include <trace/events/lock.h>
/** * queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock){ /* * Readers come here when they cannot get the lock without waiting */ if (unlikely(in_interrupt())) { /* * Readers in interrupt context will get the lock immediately * if the writer is just waiting (not holding the lock yet), * so spin with ACQUIRE semantics until the lock is available * without waiting in the queue. */ atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); return; } atomic_sub(_QR_BIAS, &lock->cnts);
trace_contention_begin(lock, LCB_F_SPIN | LCB_F_READ);
/* * Put the reader into the wait queue */ arch_spin_lock(&lock->wait_lock); atomic_add(_QR_BIAS, &lock->cnts);
/* * The ACQUIRE semantics of the following spinning code ensure * that accesses can't leak upwards out of our subsequent critical * section in the case that the lock is currently held for write. */ atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
/* * Signal the next one in queue to become queue head */ arch_spin_unlock(&lock->wait_lock);
trace_contention_end(lock, 0);}EXPORT_SYMBOL(queued_read_lock_slowpath);
/** * queued_write_lock_slowpath - acquire write lock of a queued rwlock * @lock : Pointer to queued rwlock structure */void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock){ int cnts;
trace_contention_begin(lock, LCB_F_SPIN | LCB_F_WRITE);
/* Put the writer into the wait queue */ arch_spin_lock(&lock->wait_lock);
/* Try to acquire the lock directly if no reader is present */ if (!(cnts = atomic_read(&lock->cnts)) && atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED)) goto unlock;
/* Set the waiting flag to notify readers that a writer is pending */ atomic_or(_QW_WAITING, &lock->cnts);
/* When no more readers or writers, set the locked flag */ do { cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING); } while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));unlock: arch_spin_unlock(&lock->wait_lock);
trace_contention_end(lock, 0);}EXPORT_SYMBOL(queued_write_lock_slowpath);