Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274/* SPDX-License-Identifier: GPL-2.0 *//* * allocation tagging */#ifndef _LINUX_ALLOC_TAG_H#define _LINUX_ALLOC_TAG_H
#include <linux/bug.h>#include <linux/codetag.h>#include <linux/container_of.h>#include <linux/preempt.h>#include <asm/percpu.h>#include <linux/cpumask.h>#include <linux/smp.h>#include <linux/static_key.h>#include <linux/irqflags.h>
struct alloc_tag_counters { u64 bytes; u64 calls;};
/* * An instance of this structure is created in a special ELF section at every * allocation callsite. At runtime, the special section is treated as * an array of these. Embedded codetag utilizes codetag framework. */struct alloc_tag { struct codetag ct; struct alloc_tag_counters __percpu *counters;} __aligned(8);
struct alloc_tag_kernel_section { struct alloc_tag *first_tag; unsigned long count;};
struct alloc_tag_module_section { union { unsigned long start_addr; struct alloc_tag *first_tag; }; unsigned long end_addr; /* used size */ unsigned long size;};
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
#define CODETAG_EMPTY ((void *)1)
static inline bool is_codetag_empty(union codetag_ref *ref){ return ref->ct == CODETAG_EMPTY;}
static inline void set_codetag_empty(union codetag_ref *ref){ if (ref) ref->ct = CODETAG_EMPTY;}
#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
static inline bool is_codetag_empty(union codetag_ref *ref) { return false; }
static inline void set_codetag_empty(union codetag_ref *ref){ if (ref) ref->ct = NULL;}
#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
#ifdef CONFIG_MEM_ALLOC_PROFILING
#define ALLOC_TAG_SECTION_NAME "alloc_tags"
struct codetag_bytes { struct codetag *ct; s64 bytes;};
size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep);
static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct){ return container_of(ct, struct alloc_tag, ct);}
#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)/* * When percpu variables are required to be defined as weak, static percpu * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION). * Instead we will account all module allocations to a single counter. */DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
#define DEFINE_ALLOC_TAG(_alloc_tag) \ static struct alloc_tag _alloc_tag __used __aligned(8) \ __section(ALLOC_TAG_SECTION_NAME) = { \ .ct = CODE_TAG_INIT, \ .counters = &_shared_alloc_tag };
#else /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */
#ifdef MODULE
#define DEFINE_ALLOC_TAG(_alloc_tag) \ static struct alloc_tag _alloc_tag __used __aligned(8) \ __section(ALLOC_TAG_SECTION_NAME) = { \ .ct = CODE_TAG_INIT, \ .counters = NULL };
#else /* MODULE */
#define DEFINE_ALLOC_TAG(_alloc_tag) \ static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \ static struct alloc_tag _alloc_tag __used __aligned(8) \ __section(ALLOC_TAG_SECTION_NAME) = { \ .ct = CODE_TAG_INIT, \ .counters = &_alloc_tag_cntr };
#endif /* MODULE */
#endif /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */
DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, mem_alloc_profiling_key);
static inline bool mem_alloc_profiling_enabled(void){ return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, &mem_alloc_profiling_key);}
bool mem_alloc_profiling_permanently_disabled(void);
static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag){ struct alloc_tag_counters v = { 0, 0 }; struct alloc_tag_counters *counter; int cpu;
for_each_possible_cpu(cpu) { counter = per_cpu_ptr(tag->counters, cpu); v.bytes += counter->bytes; v.calls += counter->calls; }
return v;}
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUGstatic inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag){ WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref), "alloc_tag was not cleared (got tag for %s:%u)\n", ref->ct->filename, ref->ct->lineno);
WARN_ONCE(!tag, "current->alloc_tag not set\n");}
static inline void alloc_tag_sub_check(union codetag_ref *ref){ WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");}void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags);#elsestatic inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}static inline void alloc_tag_sub_check(union codetag_ref *ref) {}static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {}#endif
/* Caller should verify both ref and tag to be valid */static inline bool __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag){ alloc_tag_add_check(ref, tag); if (!ref || !tag) return false;
ref->ct = &tag->ct; return true;}
static inline bool alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag){ if (unlikely(!__alloc_tag_ref_set(ref, tag))) return false;
/* * We need in increment the call counter every time we have a new * allocation or when we split a large allocation into smaller ones. * Each new reference for every sub-allocation needs to increment call * counter because when we free each part the counter will be decremented. */ this_cpu_inc(tag->counters->calls); return true;}
static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes){ if (likely(alloc_tag_ref_set(ref, tag))) this_cpu_add(tag->counters->bytes, bytes);}
static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes){ struct alloc_tag *tag;
alloc_tag_sub_check(ref); if (!ref || !ref->ct) return;
if (is_codetag_empty(ref)) { ref->ct = NULL; return; }
tag = ct_to_alloc_tag(ref->ct);
this_cpu_sub(tag->counters->bytes, bytes); this_cpu_dec(tag->counters->calls);
ref->ct = NULL;}
static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag){ tag->ct.flags |= CODETAG_FLAG_INACCURATE;}
static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag){ return !!(tag->ct.flags & CODETAG_FLAG_INACCURATE);}
#define alloc_tag_record(p) ((p) = current->alloc_tag)
#else /* CONFIG_MEM_ALLOC_PROFILING */
#define DEFINE_ALLOC_TAG(_alloc_tag)static inline bool mem_alloc_profiling_enabled(void) { return false; }static inline bool mem_alloc_profiling_permanently_disabled(void) { return true; }static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes) {}static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag) {}static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag) { return false; }#define alloc_tag_record(p) do {} while (0)
#endif /* CONFIG_MEM_ALLOC_PROFILING */
#define alloc_hooks_tag(_tag, _do_alloc) \({ \ typeof(_do_alloc) _res; \ if (mem_alloc_profiling_enabled()) { \ struct alloc_tag * __maybe_unused _old; \ _old = alloc_tag_save(_tag); \ _res = _do_alloc; \ alloc_tag_restore(_tag, _old); \ } else \ _res = _do_alloc; \ _res; \})
#define alloc_hooks(_do_alloc) \({ \ DEFINE_ALLOC_TAG(_alloc_tag); \ alloc_hooks_tag(&_alloc_tag, _do_alloc); \})
#endif /* _LINUX_ALLOC_TAG_H */