Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
7.6 kB · 306 lines
C
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307/* SPDX-License-Identifier: GPL-2.0-or-later *//* * Generic barrier definitions. * * It should be possible to use these on really simple architectures, * but it serves more as a starting point for new ports. * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */#ifndef __ASM_GENERIC_BARRIER_H#define __ASM_GENERIC_BARRIER_H
#ifndef __ASSEMBLY__
#include <linux/compiler.h>#include <linux/kcsan-checks.h>#include <asm/rwonce.h>
#ifndef nop#define nop() asm volatile ("nop")#endif
/* * Architectures that want generic instrumentation can define __ prefixed * variants of all barriers. */
#ifdef __mb#define mb() do { kcsan_mb(); __mb(); } while (0)#endif
#ifdef __rmb#define rmb() do { kcsan_rmb(); __rmb(); } while (0)#endif
#ifdef __wmb#define wmb() do { kcsan_wmb(); __wmb(); } while (0)#endif
#ifdef __dma_mb#define dma_mb() do { kcsan_mb(); __dma_mb(); } while (0)#endif
#ifdef __dma_rmb#define dma_rmb() do { kcsan_rmb(); __dma_rmb(); } while (0)#endif
#ifdef __dma_wmb#define dma_wmb() do { kcsan_wmb(); __dma_wmb(); } while (0)#endif
/* * Force strict CPU ordering. And yes, this is required on UP too when we're * talking to devices. * * Fall back to compiler barriers if nothing better is provided. */
#ifndef mb#define mb() barrier()#endif
#ifndef rmb#define rmb() mb()#endif
#ifndef wmb#define wmb() mb()#endif
#ifndef dma_mb#define dma_mb() mb()#endif
#ifndef dma_rmb#define dma_rmb() rmb()#endif
#ifndef dma_wmb#define dma_wmb() wmb()#endif
#ifndef __smp_mb#define __smp_mb() mb()#endif
#ifndef __smp_rmb#define __smp_rmb() rmb()#endif
#ifndef __smp_wmb#define __smp_wmb() wmb()#endif
#ifdef CONFIG_SMP
#ifndef smp_mb#define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0)#endif
#ifndef smp_rmb#define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)#endif
#ifndef smp_wmb#define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)#endif
#else /* !CONFIG_SMP */
#ifndef smp_mb#define smp_mb() barrier()#endif
#ifndef smp_rmb#define smp_rmb() barrier()#endif
#ifndef smp_wmb#define smp_wmb() barrier()#endif
#endif /* CONFIG_SMP */
#ifndef __smp_store_mb#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)#endif
#ifndef __smp_mb__before_atomic#define __smp_mb__before_atomic() __smp_mb()#endif
#ifndef __smp_mb__after_atomic#define __smp_mb__after_atomic() __smp_mb()#endif
#ifndef __smp_store_release#define __smp_store_release(p, v) \do { \ compiletime_assert_atomic_type(*p); \ __smp_mb(); \ WRITE_ONCE(*p, v); \} while (0)#endif
#ifndef __smp_load_acquire#define __smp_load_acquire(p) \({ \ __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ __smp_mb(); \ (typeof(*p))___p1; \})#endif
#ifdef CONFIG_SMP
#ifndef smp_store_mb#define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)#endif
#ifndef smp_mb__before_atomic#define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)#endif
#ifndef smp_mb__after_atomic#define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)#endif
#ifndef smp_store_release#define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)#endif
#ifndef smp_load_acquire#define smp_load_acquire(p) __smp_load_acquire(p)#endif
#else /* !CONFIG_SMP */
#ifndef smp_store_mb#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)#endif
#ifndef smp_mb__before_atomic#define smp_mb__before_atomic() barrier()#endif
#ifndef smp_mb__after_atomic#define smp_mb__after_atomic() barrier()#endif
#ifndef smp_store_release#define smp_store_release(p, v) \do { \ barrier(); \ WRITE_ONCE(*p, v); \} while (0)#endif
#ifndef smp_load_acquire#define smp_load_acquire(p) \({ \ __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ barrier(); \ (typeof(*p))___p1; \})#endif
#endif /* CONFIG_SMP */
/* Barriers for virtual machine guests when talking to an SMP host */#define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0)#define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)#define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)#define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)#define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)#define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)#define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)#define virt_load_acquire(p) __smp_load_acquire(p)
/** * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency * * A control dependency provides a LOAD->STORE order, the additional RMB * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order, * aka. (load)-ACQUIRE. * * Architectures that do not do load speculation can have this be barrier(). */#ifndef smp_acquire__after_ctrl_dep#define smp_acquire__after_ctrl_dep() smp_rmb()#endif
/** * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees * @ptr: pointer to the variable to wait on * @cond_expr: boolean expression to wait for * * Equivalent to using READ_ONCE() on the condition variable. * * Due to C lacking lambda expressions we load the value of *ptr into a * pre-named variable @VAL to be used in @cond. */#ifndef smp_cond_load_relaxed#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ typeof(ptr) __PTR = (ptr); \ __unqual_scalar_typeof(*ptr) VAL; \ for (;;) { \ VAL = READ_ONCE(*__PTR); \ if (cond_expr) \ break; \ cpu_relax(); \ } \ (typeof(*ptr))VAL; \})#endif
/** * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering * @ptr: pointer to the variable to wait on * @cond_expr: boolean expression to wait for * * Equivalent to using smp_load_acquire() on the condition variable but employs * the control dependency of the wait to reduce the barrier on many platforms. */#ifndef smp_cond_load_acquire#define smp_cond_load_acquire(ptr, cond_expr) ({ \ __unqual_scalar_typeof(*ptr) _val; \ _val = smp_cond_load_relaxed(ptr, cond_expr); \ smp_acquire__after_ctrl_dep(); \ (typeof(*ptr))_val; \})#endif
/* * pmem_wmb() ensures that all stores for which the modification * are written to persistent storage by preceding instructions have * updated persistent storage before any data access or data transfer * caused by subsequent instructions is initiated. */#ifndef pmem_wmb#define pmem_wmb() wmb()#endif
/* * ioremap_wc() maps I/O memory as memory with write-combining attributes. For * this kind of memory accesses, the CPU may wait for prior accesses to be * merged with subsequent ones. In some situation, such wait is bad for the * performance. io_stop_wc() can be used to prevent the merging of * write-combining memory accesses before this macro with those after it. */#ifndef io_stop_wc#define io_stop_wc() do { } while (0)#endif
/* * Architectures that guarantee an implicit smp_mb() in switch_mm() * can override smp_mb__after_switch_mm. */#ifndef smp_mb__after_switch_mm# define smp_mb__after_switch_mm() smp_mb()#endif
#endif /* !__ASSEMBLY__ */#endif /* __ASM_GENERIC_BARRIER_H */