Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250/* SPDX-License-Identifier: GPL-2.0 */#ifndef _KERNEL_EVENTS_INTERNAL_H#define _KERNEL_EVENTS_INTERNAL_H
#include <linux/hardirq.h>#include <linux/uaccess.h>#include <linux/refcount.h>
/* Buffer handling */
#define RING_BUFFER_WRITABLE 0x01
struct perf_buffer { refcount_t refcount; struct rcu_head rcu_head;#ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; int page_order; /* allocation order */#endif int nr_pages; /* nr of data pages */ int overwrite; /* can overwrite itself */ int paused; /* can write into ring buffer */
atomic_t poll; /* POLL_ for wakeups */
local_t head; /* write position */ unsigned int nest; /* nested writers */ local_t events; /* event limit */ local_t wakeup; /* wakeup stamp */ local_t lost; /* nr records lost */
long watermark; /* wakeup watermark */ long aux_watermark; /* poll crap */ spinlock_t event_lock; struct list_head event_list;
refcount_t mmap_count; unsigned long mmap_locked; struct user_struct *mmap_user;
/* AUX area */ struct mutex aux_mutex; long aux_head; unsigned int aux_nest; long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ unsigned long aux_pgoff; int aux_nr_pages; int aux_overwrite; refcount_t aux_mmap_count; unsigned long aux_mmap_locked; void (*free_aux)(void *); refcount_t aux_refcount; int aux_in_sampling; int aux_in_pause_resume; void **aux_pages; void *aux_priv;
struct perf_event_mmap_page *user_page; void *data_pages[];};
extern void rb_free(struct perf_buffer *rb);
static inline void rb_free_rcu(struct rcu_head *rcu_head){ struct perf_buffer *rb;
rb = container_of(rcu_head, struct perf_buffer, rcu_head); free_uid(rb->mmap_user); rb_free(rb);}
static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause){ if (!pause && rb->nr_pages) rb->paused = 0; else rb->paused = 1;}
extern struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags);extern void perf_event_wakeup(struct perf_event *event);extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, pgoff_t pgoff, int nr_pages, long watermark, int flags);extern void rb_free_aux(struct perf_buffer *rb);extern struct perf_buffer *ring_buffer_get(struct perf_event *event);extern void ring_buffer_put(struct perf_buffer *rb);
static inline bool rb_has_aux(struct perf_buffer *rb){ return !!rb->aux_nr_pages;}
void perf_event_aux_event(struct perf_event *event, unsigned long head, unsigned long size, u64 flags);
extern struct page *perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff);
#ifdef CONFIG_PERF_USE_VMALLOC/* * Back perf_mmap() with vmalloc memory. * * Required for architectures that have d-cache aliasing issues. */
static inline int page_order(struct perf_buffer *rb){ return rb->page_order;}
#else
static inline int page_order(struct perf_buffer *rb){ return 0;}#endif
static inline int data_page_nr(struct perf_buffer *rb){ return rb->nr_pages << page_order(rb);}
static inline unsigned long perf_data_size(struct perf_buffer *rb){ return rb->nr_pages << (PAGE_SHIFT + page_order(rb));}
static inline unsigned long perf_aux_size(struct perf_buffer *rb){ return (unsigned long)rb->aux_nr_pages << PAGE_SHIFT;}
#define __DEFINE_OUTPUT_COPY_BODY(advance_buf, memcpy_func, ...) \{ \ unsigned long size, written; \ \ do { \ size = min(handle->size, len); \ written = memcpy_func(__VA_ARGS__); \ written = size - written; \ \ len -= written; \ handle->addr += written; \ if (advance_buf) \ buf += written; \ handle->size -= written; \ if (!handle->size) { \ struct perf_buffer *rb = handle->rb; \ \ handle->page++; \ handle->page &= rb->nr_pages - 1; \ handle->addr = rb->data_pages[handle->page]; \ handle->size = PAGE_SIZE << page_order(rb); \ } \ } while (len && written == size); \ \ return len; \}
#define DEFINE_OUTPUT_COPY(func_name, memcpy_func) \static inline unsigned long \func_name(struct perf_output_handle *handle, \ const void *buf, unsigned long len) \__DEFINE_OUTPUT_COPY_BODY(true, memcpy_func, handle->addr, buf, size)
static inline unsigned long__output_custom(struct perf_output_handle *handle, perf_copy_f copy_func, const void *buf, unsigned long len){ unsigned long orig_len = len; __DEFINE_OUTPUT_COPY_BODY(false, copy_func, handle->addr, buf, orig_len - len, size)}
static inline unsigned longmemcpy_common(void *dst, const void *src, unsigned long n){ memcpy(dst, src, n); return 0;}
DEFINE_OUTPUT_COPY(__output_copy, memcpy_common)
static inline unsigned longmemcpy_skip(void *dst, const void *src, unsigned long n){ return 0;}
DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip)
#ifndef arch_perf_out_copy_user#define arch_perf_out_copy_user arch_perf_out_copy_user
static inline unsigned longarch_perf_out_copy_user(void *dst, const void *src, unsigned long n){ unsigned long ret;
pagefault_disable(); ret = __copy_from_user_inatomic(dst, src, n); pagefault_enable();
return ret;}#endif
DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
static inline int get_recursion_context(u8 *recursion){ unsigned char rctx = interrupt_context_level();
if (recursion[rctx]) return -1;
recursion[rctx]++; barrier();
return rctx;}
static inline void put_recursion_context(u8 *recursion, unsigned char rctx){ barrier(); recursion[rctx]--;}
#ifdef CONFIG_HAVE_PERF_USER_STACK_DUMPstatic inline bool arch_perf_have_user_stack_dump(void){ return true;}
#define perf_user_stack_pointer(regs) user_stack_pointer(regs)#elsestatic inline bool arch_perf_have_user_stack_dump(void){ return false;}
#define perf_user_stack_pointer(regs) 0#endif /* CONFIG_HAVE_PERF_USER_STACK_DUMP */
#endif /* _KERNEL_EVENTS_INTERNAL_H */