Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590// SPDX-License-Identifier: GPL-2.0/* * Fprobe-based tracing events * Copyright (C) 2022 Google LLC. */#define pr_fmt(fmt) "trace_fprobe: " fmt
#include <linux/fprobe.h>#include <linux/list.h>#include <linux/module.h>#include <linux/mutex.h>#include <linux/rculist.h>#include <linux/security.h>#include <linux/tracepoint.h>#include <linux/uaccess.h>
#include <asm/ptrace.h>
#include "trace_dynevent.h"#include "trace_probe.h"#include "trace_probe_kernel.h"#include "trace_probe_tmpl.h"
#define FPROBE_EVENT_SYSTEM "fprobes"#define TRACEPOINT_EVENT_SYSTEM "tracepoints"#define RETHOOK_MAXACTIVE_MAX 4096
static int trace_fprobe_create(const char *raw_command);static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);static int trace_fprobe_release(struct dyn_event *ev);static bool trace_fprobe_is_busy(struct dyn_event *ev);static bool trace_fprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev);
static struct dyn_event_operations trace_fprobe_ops = { .create = trace_fprobe_create, .show = trace_fprobe_show, .is_busy = trace_fprobe_is_busy, .free = trace_fprobe_release, .match = trace_fprobe_match,};
/* List of tracepoint_user */static LIST_HEAD(tracepoint_user_list);static DEFINE_MUTEX(tracepoint_user_mutex);
/* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */struct tracepoint_user { struct list_head list; const char *name; struct tracepoint *tpoint; unsigned int refcount;};
/* NOTE: you must lock tracepoint_user_mutex. */#define for_each_tracepoint_user(tuser) \ list_for_each_entry(tuser, &tracepoint_user_list, list)
static int tracepoint_user_register(struct tracepoint_user *tuser){ struct tracepoint *tpoint = tuser->tpoint;
if (!tpoint) return 0;
return tracepoint_probe_register_prio_may_exist(tpoint, tpoint->probestub, NULL, 0);}
static void tracepoint_user_unregister(struct tracepoint_user *tuser){ if (!tuser->tpoint) return;
WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL)); tuser->tpoint = NULL;}
static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser){ if (!tuser->tpoint) return 0UL;
return (unsigned long)tuser->tpoint->probestub;}
static void __tracepoint_user_free(struct tracepoint_user *tuser){ if (!tuser) return; kfree(tuser->name); kfree(tuser);}
DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T))
static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint){ struct tracepoint_user *tuser __free(tuser_free) = NULL; int ret;
tuser = kzalloc_obj(*tuser); if (!tuser) return NULL; tuser->name = kstrdup(name, GFP_KERNEL); if (!tuser->name) return NULL;
/* Register tracepoint if it is loaded. */ if (tpoint) { tuser->tpoint = tpoint; ret = tracepoint_user_register(tuser); if (ret) return ERR_PTR(ret); }
tuser->refcount = 1; INIT_LIST_HEAD(&tuser->list); list_add(&tuser->list, &tracepoint_user_list);
return_ptr(tuser);}
static struct tracepoint *find_tracepoint(const char *tp_name, struct module **tp_mod);
/* * Get tracepoint_user if exist, or allocate new one and register it. * If tracepoint is on a module, get its refcounter too. * This returns errno or NULL (not loaded yet) or tracepoint_user. */static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod){ struct module *mod __free(module_put) = NULL; struct tracepoint_user *tuser; struct tracepoint *tpoint;
if (!name || !pmod) return ERR_PTR(-EINVAL);
/* Get and lock the module which has tracepoint. */ tpoint = find_tracepoint(name, &mod);
guard(mutex)(&tracepoint_user_mutex); /* Search existing tracepoint_user */ for_each_tracepoint_user(tuser) { if (!strcmp(tuser->name, name)) { tuser->refcount++; *pmod = no_free_ptr(mod); return tuser; } }
/* The corresponding tracepoint_user is not found. */ tuser = __tracepoint_user_init(name, tpoint); if (!IS_ERR_OR_NULL(tuser)) *pmod = no_free_ptr(mod);
return tuser;}
static void tracepoint_user_put(struct tracepoint_user *tuser){ scoped_guard(mutex, &tracepoint_user_mutex) { if (--tuser->refcount > 0) return;
list_del(&tuser->list); tracepoint_user_unregister(tuser); }
__tracepoint_user_free(tuser);}
DEFINE_FREE(tuser_put, struct tracepoint_user *, if (!IS_ERR_OR_NULL(_T)) tracepoint_user_put(_T))
/* * Fprobe event core functions */
/* * @tprobe is true for tracepoint probe. * @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not * loaded with a module. If @tuser != NULL, this trace_fprobe is enabled. */struct trace_fprobe { struct dyn_event devent; struct fprobe fp; const char *symbol; bool tprobe; struct tracepoint_user *tuser; struct trace_probe tp;};
static bool is_trace_fprobe(struct dyn_event *ev){ return ev->ops == &trace_fprobe_ops;}
static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev){ return container_of(ev, struct trace_fprobe, devent);}
/** * for_each_trace_fprobe - iterate over the trace_fprobe list * @pos: the struct trace_fprobe * for each entry * @dpos: the struct dyn_event * to use as a loop cursor */#define for_each_trace_fprobe(pos, dpos) \ for_each_dyn_event(dpos) \ if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
static bool trace_fprobe_is_return(struct trace_fprobe *tf){ return tf->fp.exit_handler != NULL;}
static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf){ return tf->tprobe;}
static const char *trace_fprobe_symbol(struct trace_fprobe *tf){ return tf->symbol ? tf->symbol : "unknown";}
static bool trace_fprobe_is_busy(struct dyn_event *ev){ struct trace_fprobe *tf = to_trace_fprobe(ev);
return trace_probe_is_enabled(&tf->tp);}
static bool trace_fprobe_match_command_head(struct trace_fprobe *tf, int argc, const char **argv){ char buf[MAX_ARGSTR_LEN + 1];
if (!argc) return true;
snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf)); if (strcmp(buf, argv[0])) return false; argc--; argv++;
return trace_probe_match_command_args(&tf->tp, argc, argv);}
static bool trace_fprobe_match(const char *system, const char *event, int argc, const char **argv, struct dyn_event *ev){ struct trace_fprobe *tf = to_trace_fprobe(ev);
if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event)) return false;
if (system && strcmp(trace_probe_group_name(&tf->tp), system)) return false;
return trace_fprobe_match_command_head(tf, argc, argv);}
static bool trace_fprobe_is_registered(struct trace_fprobe *tf){ return fprobe_is_registered(&tf->fp);}
/* * Note that we don't verify the fetch_insn code, since it does not come * from user space. */static intprocess_fetch_insn(struct fetch_insn *code, void *rec, void *edata, void *dest, void *base){ struct ftrace_regs *fregs = rec; unsigned long val; int ret;
retry: /* 1st stage: get value from context */ switch (code->op) { case FETCH_OP_STACK: val = ftrace_regs_get_kernel_stack_nth(fregs, code->param); break; case FETCH_OP_STACKP: val = ftrace_regs_get_stack_pointer(fregs); break; case FETCH_OP_RETVAL: val = ftrace_regs_get_return_value(fregs); break;#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API case FETCH_OP_ARG: val = ftrace_regs_get_argument(fregs, code->param); break; case FETCH_OP_EDATA: val = *(unsigned long *)((unsigned long)edata + code->offset); break;#endif case FETCH_NOP_SYMBOL: /* Ignore a place holder */ code++; goto retry; default: ret = process_common_fetch_insn(code, &val); if (ret < 0) return ret; } code++;
return process_fetch_insn_bottom(code, val, dest, base);}NOKPROBE_SYMBOL(process_fetch_insn)
/* function entry handler */static nokprobe_inline void__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs, struct trace_event_file *trace_file){ struct fentry_trace_entry_head *entry; struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct trace_event_buffer fbuffer; int dsize;
if (WARN_ON_ONCE(call != trace_file->event_call)) return;
if (trace_trigger_soft_disabled(trace_file)) return;
dsize = __get_data_size(&tf->tp, fregs, NULL);
entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry) + tf->tp.size + dsize); if (!entry) return;
fbuffer.regs = ftrace_get_regs(fregs); entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); entry->ip = entry_ip; store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);
trace_event_buffer_commit(&fbuffer);}
static voidfentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs){ struct event_file_link *link;
trace_probe_for_each_link_rcu(link, &tf->tp) __fentry_trace_func(tf, entry_ip, fregs, link->file);}NOKPROBE_SYMBOL(fentry_trace_func);
static nokprobe_inlinevoid store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs){ struct probe_entry_arg *earg = tp->entry_arg; unsigned long val = 0; int i;
if (!earg) return;
for (i = 0; i < earg->size; i++) { struct fetch_insn *code = &earg->code[i];
switch (code->op) { case FETCH_OP_ARG: val = ftrace_regs_get_argument(fregs, code->param); break; case FETCH_OP_ST_EDATA: *(unsigned long *)((unsigned long)edata + code->offset) = val; break; case FETCH_OP_END: goto end; default: break; } }end: return;}
/* function exit handler */static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
if (tf->tp.entry_arg) store_fprobe_entry_data(entry_data, &tf->tp, fregs);
return 0;}NOKPROBE_SYMBOL(trace_fprobe_entry_handler)
static nokprobe_inline void__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data, struct trace_event_file *trace_file){ struct fexit_trace_entry_head *entry; struct trace_event_buffer fbuffer; struct trace_event_call *call = trace_probe_event_call(&tf->tp); int dsize;
if (WARN_ON_ONCE(call != trace_file->event_call)) return;
if (trace_trigger_soft_disabled(trace_file)) return;
dsize = __get_data_size(&tf->tp, fregs, entry_data);
entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry) + tf->tp.size + dsize); if (!entry) return;
fbuffer.regs = ftrace_get_regs(fregs); entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); entry->func = entry_ip; entry->ret_ip = ret_ip; store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);
trace_event_buffer_commit(&fbuffer);}
static voidfexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct event_file_link *link;
trace_probe_for_each_link_rcu(link, &tf->tp) __fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file);}NOKPROBE_SYMBOL(fexit_trace_func);
#ifdef CONFIG_PERF_EVENTS
static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, struct ftrace_regs *fregs){ struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct fentry_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; struct pt_regs *regs; int rctx;
head = this_cpu_ptr(call->perf_events); if (hlist_empty(head)) return 0;
dsize = __get_data_size(&tf->tp, fregs, NULL); __size = sizeof(*entry) + tf->tp.size + dsize; size = ALIGN(__size + sizeof(u32), sizeof(u64)); size -= sizeof(u32);
entry = perf_trace_buf_alloc(size, ®s, &rctx); if (!entry) return 0;
regs = ftrace_fill_perf_regs(fregs, regs);
entry->ip = entry_ip; memset(&entry[1], 0, dsize); store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, head, NULL); return 0;}NOKPROBE_SYMBOL(fentry_perf_func);
static voidfexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct trace_event_call *call = trace_probe_event_call(&tf->tp); struct fexit_trace_entry_head *entry; struct hlist_head *head; int size, __size, dsize; struct pt_regs *regs; int rctx;
head = this_cpu_ptr(call->perf_events); if (hlist_empty(head)) return;
dsize = __get_data_size(&tf->tp, fregs, entry_data); __size = sizeof(*entry) + tf->tp.size + dsize; size = ALIGN(__size + sizeof(u32), sizeof(u64)); size -= sizeof(u32);
entry = perf_trace_buf_alloc(size, ®s, &rctx); if (!entry) return;
regs = ftrace_fill_perf_regs(fregs, regs);
entry->func = entry_ip; entry->ret_ip = ret_ip; store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize); perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, head, NULL);}NOKPROBE_SYMBOL(fexit_perf_func);#endif /* CONFIG_PERF_EVENTS */
static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); unsigned int flags = trace_probe_load_flag(&tf->tp); int ret = 0;
if (flags & TP_FLAG_TRACE) fentry_trace_func(tf, entry_ip, fregs);
#ifdef CONFIG_PERF_EVENTS if (flags & TP_FLAG_PROFILE) ret = fentry_perf_func(tf, entry_ip, fregs);#endif return ret;}NOKPROBE_SYMBOL(fentry_dispatcher);
static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip, unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data){ struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); unsigned int flags = trace_probe_load_flag(&tf->tp);
if (flags & TP_FLAG_TRACE) fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data);#ifdef CONFIG_PERF_EVENTS if (flags & TP_FLAG_PROFILE) fexit_perf_func(tf, entry_ip, ret_ip, fregs, entry_data);#endif}NOKPROBE_SYMBOL(fexit_dispatcher);
static void free_trace_fprobe(struct trace_fprobe *tf){ if (tf) { trace_probe_cleanup(&tf->tp); if (tf->tuser) tracepoint_user_put(tf->tuser); kfree(tf->symbol); kfree(tf); }}
/* Since alloc_trace_fprobe() can return error, check the pointer is ERR too. */DEFINE_FREE(free_trace_fprobe, struct trace_fprobe *, if (!IS_ERR_OR_NULL(_T)) free_trace_fprobe(_T))
/* * Allocate new trace_probe and initialize it (including fprobe). */static struct trace_fprobe *alloc_trace_fprobe(const char *group, const char *event, const char *symbol, int nargs, bool is_return, bool is_tracepoint){ struct trace_fprobe *tf __free(free_trace_fprobe) = NULL; int ret = -ENOMEM;
tf = kzalloc_flex(*tf, tp.args, nargs); if (!tf) return ERR_PTR(ret);
tf->symbol = kstrdup(symbol, GFP_KERNEL); if (!tf->symbol) return ERR_PTR(-ENOMEM);
if (is_return) tf->fp.exit_handler = fexit_dispatcher; else tf->fp.entry_handler = fentry_dispatcher;
tf->tprobe = is_tracepoint;
ret = trace_probe_init(&tf->tp, event, group, false, nargs); if (ret < 0) return ERR_PTR(ret);
dyn_event_init(&tf->devent, &trace_fprobe_ops); return_ptr(tf);}
static struct trace_fprobe *find_trace_fprobe(const char *event, const char *group){ struct dyn_event *pos; struct trace_fprobe *tf;
for_each_trace_fprobe(tf, pos) if (strcmp(trace_probe_name(&tf->tp), event) == 0 && strcmp(trace_probe_group_name(&tf->tp), group) == 0) return tf; return NULL;}
/* Event entry printers */static enum print_line_tprint_fentry_event(struct trace_iterator *iter, int flags, struct trace_event *event){ struct fentry_trace_entry_head *field; struct trace_seq *s = &iter->seq; struct trace_probe *tp;
field = (struct fentry_trace_entry_head *)iter->ent; tp = trace_probe_primary_from_call( container_of(event, struct trace_event_call, event)); if (WARN_ON_ONCE(!tp)) goto out;
trace_seq_printf(s, "%s: (", trace_probe_name(tp));
if (!seq_print_ip_sym_offset(s, field->ip, flags)) goto out;
trace_seq_putc(s, ')');
if (trace_probe_print_args(s, tp->args, tp->nr_args, (u8 *)&field[1], field) < 0) goto out;
trace_seq_putc(s, '\n'); out: return trace_handle_return(s);}
static enum print_line_tprint_fexit_event(struct trace_iterator *iter, int flags, struct trace_event *event){ struct fexit_trace_entry_head *field; struct trace_seq *s = &iter->seq; struct trace_probe *tp;
field = (struct fexit_trace_entry_head *)iter->ent; tp = trace_probe_primary_from_call( container_of(event, struct trace_event_call, event)); if (WARN_ON_ONCE(!tp)) goto out;
trace_seq_printf(s, "%s: (", trace_probe_name(tp));
if (!seq_print_ip_sym_offset(s, field->ret_ip, flags)) goto out;
trace_seq_puts(s, " <- ");
if (!seq_print_ip_sym_no_offset(s, field->func, flags)) goto out;
trace_seq_putc(s, ')');
if (trace_probe_print_args(s, tp->args, tp->nr_args, (u8 *)&field[1], field) < 0) goto out;
trace_seq_putc(s, '\n');
out: return trace_handle_return(s);}
static int fentry_event_define_fields(struct trace_event_call *event_call){ int ret; struct fentry_trace_entry_head field; struct trace_probe *tp;
tp = trace_probe_primary_from_call(event_call); if (WARN_ON_ONCE(!tp)) return -ENOENT;
DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
return traceprobe_define_arg_fields(event_call, sizeof(field), tp);}
static int fexit_event_define_fields(struct trace_event_call *event_call){ int ret; struct fexit_trace_entry_head field; struct trace_probe *tp;
tp = trace_probe_primary_from_call(event_call); if (WARN_ON_ONCE(!tp)) return -ENOENT;
DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
return traceprobe_define_arg_fields(event_call, sizeof(field), tp);}
static struct trace_event_functions fentry_funcs = { .trace = print_fentry_event};
static struct trace_event_functions fexit_funcs = { .trace = print_fexit_event};
static struct trace_event_fields fentry_fields_array[] = { { .type = TRACE_FUNCTION_TYPE, .define_fields = fentry_event_define_fields }, {}};
static struct trace_event_fields fexit_fields_array[] = { { .type = TRACE_FUNCTION_TYPE, .define_fields = fexit_event_define_fields }, {}};
static int fprobe_register(struct trace_event_call *event, enum trace_reg type, void *data);
static inline void init_trace_event_call(struct trace_fprobe *tf){ struct trace_event_call *call = trace_probe_event_call(&tf->tp);
if (trace_fprobe_is_return(tf)) { call->event.funcs = &fexit_funcs; call->class->fields_array = fexit_fields_array; } else { call->event.funcs = &fentry_funcs; call->class->fields_array = fentry_fields_array; }
call->flags = TRACE_EVENT_FL_FPROBE; call->class->reg = fprobe_register;}
static int register_fprobe_event(struct trace_fprobe *tf){ init_trace_event_call(tf);
return trace_probe_register_event_call(&tf->tp);}
static int unregister_fprobe_event(struct trace_fprobe *tf){ return trace_probe_unregister_event_call(&tf->tp);}
static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf){ struct tracepoint_user *tuser __free(tuser_put) = NULL; struct module *mod __free(module_put) = NULL; unsigned long ip; int ret;
if (WARN_ON_ONCE(tf->tuser)) return -EINVAL;
/* If the tracepoint is in a module, it must be locked in this function. */ tuser = tracepoint_user_find_get(tf->symbol, &mod); /* This tracepoint is not loaded yet */ if (IS_ERR(tuser)) return PTR_ERR(tuser); if (!tuser) return -ENOMEM;
/* Register fprobe only if the tracepoint is loaded. */ if (tuser->tpoint) { ip = tracepoint_user_ip(tuser); if (WARN_ON_ONCE(!ip)) return -ENOENT;
ret = register_fprobe_ips(&tf->fp, &ip, 1); if (ret < 0) return ret; }
tf->tuser = no_free_ptr(tuser); return 0;}
/* Returns an error if the target function is not available, or 0 */static int trace_fprobe_verify_target(struct trace_fprobe *tf){ int ret;
/* Tracepoint should have a stub function. */ if (trace_fprobe_is_tracepoint(tf)) return 0;
/* * Note: since we don't lock the module, even if this succeeded, * register_fprobe() later can fail. */ ret = fprobe_count_ips_from_filter(tf->symbol, NULL); return (ret < 0) ? ret : 0;}
/* Internal register function - just handle fprobe and flags */static int __register_trace_fprobe(struct trace_fprobe *tf){ int i, ret;
/* Should we need new LOCKDOWN flag for fprobe? */ ret = security_locked_down(LOCKDOWN_KPROBES); if (ret) return ret;
if (trace_fprobe_is_registered(tf)) return -EINVAL;
for (i = 0; i < tf->tp.nr_args; i++) { ret = traceprobe_update_arg(&tf->tp.args[i]); if (ret) return ret; }
tf->fp.flags &= ~FPROBE_FL_DISABLED;
if (trace_fprobe_is_tracepoint(tf)) return __regsiter_tracepoint_fprobe(tf);
/* TODO: handle filter, nofilter or symbol list */ return register_fprobe(&tf->fp, tf->symbol, NULL);}
/* Internal unregister function - just handle fprobe and flags */static void __unregister_trace_fprobe(struct trace_fprobe *tf){ if (trace_fprobe_is_registered(tf)) unregister_fprobe(&tf->fp); if (tf->tuser) { tracepoint_user_put(tf->tuser); tf->tuser = NULL; }}
/* TODO: make this trace_*probe common function *//* Unregister a trace_probe and probe_event */static int unregister_trace_fprobe(struct trace_fprobe *tf){ /* If other probes are on the event, just unregister fprobe */ if (trace_probe_has_sibling(&tf->tp)) goto unreg;
/* Enabled event can not be unregistered */ if (trace_probe_is_enabled(&tf->tp)) return -EBUSY;
/* If there's a reference to the dynamic event */ if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp))) return -EBUSY;
/* Will fail if probe is being used by ftrace or perf */ if (unregister_fprobe_event(tf)) return -EBUSY;
unreg: __unregister_trace_fprobe(tf); dyn_event_remove(&tf->devent); trace_probe_unlink(&tf->tp);
return 0;}
static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig, struct trace_fprobe *comp){ struct trace_probe_event *tpe = orig->tp.event; int i;
list_for_each_entry(orig, &tpe->probes, tp.list) { if (strcmp(trace_fprobe_symbol(orig), trace_fprobe_symbol(comp))) continue;
/* * trace_probe_compare_arg_type() ensured that nr_args and * each argument name and type are same. Let's compare comm. */ for (i = 0; i < orig->tp.nr_args; i++) { if (strcmp(orig->tp.args[i].comm, comp->tp.args[i].comm)) break; }
if (i == orig->tp.nr_args) return true; }
return false;}
static int append_trace_fprobe_event(struct trace_fprobe *tf, struct trace_fprobe *to){ int ret;
if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) || trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) { trace_probe_log_set_index(0); trace_probe_log_err(0, DIFF_PROBE_TYPE); return -EEXIST; } ret = trace_probe_compare_arg_type(&tf->tp, &to->tp); if (ret) { /* Note that argument starts index = 2 */ trace_probe_log_set_index(ret + 1); trace_probe_log_err(0, DIFF_ARG_TYPE); return -EEXIST; } if (trace_fprobe_has_same_fprobe(to, tf)) { trace_probe_log_set_index(0); trace_probe_log_err(0, SAME_PROBE); return -EEXIST; }
/* Append to existing event */ ret = trace_probe_append(&tf->tp, &to->tp); if (ret) return ret;
ret = trace_fprobe_verify_target(tf); if (ret) trace_probe_unlink(&tf->tp); else dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
return ret;}
/* Register a trace_probe and probe_event, and check the fprobe is available. */static int register_trace_fprobe_event(struct trace_fprobe *tf){ struct trace_fprobe *old_tf; int ret;
guard(mutex)(&event_mutex);
old_tf = find_trace_fprobe(trace_probe_name(&tf->tp), trace_probe_group_name(&tf->tp)); if (old_tf) return append_trace_fprobe_event(tf, old_tf);
/* Register new event */ ret = register_fprobe_event(tf); if (ret) { if (ret == -EEXIST) { trace_probe_log_set_index(0); trace_probe_log_err(0, EVENT_EXIST); } else pr_warn("Failed to register probe event(%d)\n", ret); return ret; }
/* Verify fprobe is sane. */ ret = trace_fprobe_verify_target(tf); if (ret < 0) unregister_fprobe_event(tf); else dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
return ret;}
struct __find_tracepoint_cb_data { const char *tp_name; struct tracepoint *tpoint; struct module *mod;};
static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv){ struct __find_tracepoint_cb_data *data = priv;
if (!data->tpoint && !strcmp(data->tp_name, tp->name)) { /* If module is not specified, try getting module refcount. */ if (!data->mod && mod) { /* If failed to get refcount, ignore this tracepoint. */ if (!try_module_get(mod)) return;
data->mod = mod; } data->tpoint = tp; }}
static void __find_tracepoint_cb(struct tracepoint *tp, void *priv){ struct __find_tracepoint_cb_data *data = priv;
if (!data->tpoint && !strcmp(data->tp_name, tp->name)) data->tpoint = tp;}
/* * Find a tracepoint from kernel and module. If the tracepoint is on the module, * the module's refcount is incremented and returned as *@tp_mod. Thus, if it is * not NULL, caller must call module_put(*tp_mod) after used the tracepoint. */static struct tracepoint *find_tracepoint(const char *tp_name, struct module **tp_mod){ struct __find_tracepoint_cb_data data = { .tp_name = tp_name, .mod = NULL, };
for_each_kernel_tracepoint(__find_tracepoint_cb, &data);
if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) { for_each_module_tracepoint(__find_tracepoint_module_cb, &data); *tp_mod = data.mod; }
return data.tpoint;}
#ifdef CONFIG_MODULES/* * Find a tracepoint from specified module. In this case, this does not get the * module's refcount. The caller must ensure the module is not freed. */static struct tracepoint *find_tracepoint_in_module(struct module *mod, const char *tp_name){ struct __find_tracepoint_cb_data data = { .tp_name = tp_name, .mod = mod, };
for_each_tracepoint_in_module(mod, __find_tracepoint_module_cb, &data); return data.tpoint;}
/* These are CONFIG_MODULES=y specific functions. */static bool tracepoint_user_within_module(struct tracepoint_user *tuser, struct module *mod){ return within_module(tracepoint_user_ip(tuser), mod);}
static int tracepoint_user_register_again(struct tracepoint_user *tuser, struct tracepoint *tpoint){ tuser->tpoint = tpoint; return tracepoint_user_register(tuser);}
static void tracepoint_user_unregister_clear(struct tracepoint_user *tuser){ tracepoint_user_unregister(tuser); tuser->tpoint = NULL;}
/* module callback for tracepoint_user */static int __tracepoint_probe_module_cb(struct notifier_block *self, unsigned long val, void *data){ struct tp_module *tp_mod = data; struct tracepoint_user *tuser; struct tracepoint *tpoint;
if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING) return NOTIFY_DONE;
mutex_lock(&tracepoint_user_mutex); for_each_tracepoint_user(tuser) { if (val == MODULE_STATE_COMING) { /* This is not a tracepoint in this module. Skip it. */ tpoint = find_tracepoint_in_module(tp_mod->mod, tuser->name); if (!tpoint) continue; WARN_ON_ONCE(tracepoint_user_register_again(tuser, tpoint)); } else if (val == MODULE_STATE_GOING && tracepoint_user_within_module(tuser, tp_mod->mod)) { /* Unregister all tracepoint_user in this module. */ tracepoint_user_unregister_clear(tuser); } } mutex_unlock(&tracepoint_user_mutex);
return NOTIFY_DONE;}
static struct notifier_block tracepoint_module_nb = { .notifier_call = __tracepoint_probe_module_cb,};
/* module callback for tprobe events */static int __tprobe_event_module_cb(struct notifier_block *self, unsigned long val, void *data){ struct trace_fprobe *tf; struct dyn_event *pos; struct module *mod = data;
if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING) return NOTIFY_DONE;
mutex_lock(&event_mutex); for_each_trace_fprobe(tf, pos) { /* Skip fprobe and disabled tprobe events. */ if (!trace_fprobe_is_tracepoint(tf) || !tf->tuser) continue;
/* Before this notification, tracepoint notifier has already done. */ if (val == MODULE_STATE_COMING && tracepoint_user_within_module(tf->tuser, mod)) { unsigned long ip = tracepoint_user_ip(tf->tuser);
WARN_ON_ONCE(register_fprobe_ips(&tf->fp, &ip, 1)); } else if (val == MODULE_STATE_GOING && /* * tracepoint_user_within_module() does not work here because * tracepoint_user is already unregistered and cleared tpoint. * Instead, checking whether the fprobe is registered but * tpoint is cleared(unregistered). Such unbalance probes * must be adjusted anyway. */ trace_fprobe_is_registered(tf) && !tf->tuser->tpoint) { unregister_fprobe(&tf->fp); } } mutex_unlock(&event_mutex);
return NOTIFY_DONE;}
/* NOTE: this must be called after tracepoint callback */static struct notifier_block tprobe_event_module_nb = { .notifier_call = __tprobe_event_module_cb, /* Make sure this is later than tracepoint module notifier. */ .priority = -10,};#endif /* CONFIG_MODULES */
static int parse_symbol_and_return(int argc, const char *argv[], char **symbol, bool *is_return, bool is_tracepoint){ char *tmp = strchr(argv[1], '%'); int i;
if (tmp) { int len = tmp - argv[1];
if (!is_tracepoint && !strcmp(tmp, "%return")) { *is_return = true; } else { trace_probe_log_err(len, BAD_ADDR_SUFFIX); return -EINVAL; } *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL); } else *symbol = kstrdup(argv[1], GFP_KERNEL); if (!*symbol) return -ENOMEM;
if (*is_return) return 0;
if (is_tracepoint) { tmp = *symbol; while (*tmp && (isalnum(*tmp) || *tmp == '_')) tmp++; if (*tmp) { /* find a wrong character. */ trace_probe_log_err(tmp - *symbol, BAD_TP_NAME); kfree(*symbol); *symbol = NULL; return -EINVAL; } }
/* If there is $retval, this should be a return fprobe. */ for (i = 2; i < argc; i++) { tmp = strstr(argv[i], "$retval"); if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') { if (is_tracepoint) { trace_probe_log_set_index(i); trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE); kfree(*symbol); *symbol = NULL; return -EINVAL; } *is_return = true; break; } } return 0;}
static int trace_fprobe_create_internal(int argc, const char *argv[], struct traceprobe_parse_context *ctx){ /* * Argument syntax: * - Add fentry probe: * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS] * - Add fexit probe: * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS] * - Add tracepoint probe: * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS] * * Fetch args: * $retval : fetch return value * $stack : fetch stack address * $stackN : fetch Nth entry of stack (N:0-) * $argN : fetch Nth argument (N:1-) * $comm : fetch current task comm * @ADDR : fetch memory at ADDR (ADDR should be in kernel) * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) * Dereferencing memory fetch: * +|-offs(ARG) : fetch memory at ARG +|- offs address. * Alias name of args: * NAME=FETCHARG : set NAME as alias of FETCHARG. * Type of args: * FETCHARG:TYPE : use TYPE instead of unsigned long. */ struct trace_fprobe *tf __free(free_trace_fprobe) = NULL; const char *event = NULL, *group = FPROBE_EVENT_SYSTEM; struct module *mod __free(module_put) = NULL; const char **new_argv __free(kfree) = NULL; char *symbol __free(kfree) = NULL; char *ebuf __free(kfree) = NULL; char *gbuf __free(kfree) = NULL; char *sbuf __free(kfree) = NULL; char *abuf __free(kfree) = NULL; char *dbuf __free(kfree) = NULL; int i, new_argc = 0, ret = 0; bool is_tracepoint = false; bool is_return = false;
if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2) return -ECANCELED;
if (argv[0][0] == 't') { is_tracepoint = true; group = TRACEPOINT_EVENT_SYSTEM; }
if (argv[0][1] != '\0') { if (argv[0][1] != ':') { trace_probe_log_set_index(0); trace_probe_log_err(1, BAD_MAXACT); return -EINVAL; } event = &argv[0][2]; }
trace_probe_log_set_index(1);
/* a symbol(or tracepoint) must be specified */ ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint); if (ret < 0) return -EINVAL;
trace_probe_log_set_index(0); if (event) { gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); if (!gbuf) return -ENOMEM; ret = traceprobe_parse_event_name(&event, &group, gbuf, event - argv[0]); if (ret) return -EINVAL; }
if (!event) { ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); if (!ebuf) return -ENOMEM; /* Make a new event name */ if (is_tracepoint) snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s%s", isdigit(*symbol) ? "_" : "", symbol); else snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s__%s", symbol, is_return ? "exit" : "entry"); sanitize_event_name(ebuf); event = ebuf; }
if (is_return) ctx->flags |= TPARG_FL_RETURN; else ctx->flags |= TPARG_FL_FENTRY;
ctx->funcname = NULL; if (is_tracepoint) { /* Get tracepoint and lock its module until the end of the registration. */ struct tracepoint *tpoint;
ctx->flags |= TPARG_FL_TPOINT; mod = NULL; tpoint = find_tracepoint(symbol, &mod); if (tpoint) { sbuf = kmalloc(KSYM_NAME_LEN, GFP_KERNEL); if (!sbuf) return -ENOMEM; ctx->funcname = kallsyms_lookup((unsigned long)tpoint->probestub, NULL, NULL, NULL, sbuf); } } if (!ctx->funcname) ctx->funcname = symbol;
abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL); if (!abuf) return -ENOMEM; argc -= 2; argv += 2; new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, abuf, MAX_BTF_ARGS_LEN, ctx); if (IS_ERR(new_argv)) return PTR_ERR(new_argv); if (new_argv) { argc = new_argc; argv = new_argv; } if (argc > MAX_TRACE_ARGS) { trace_probe_log_set_index(2); trace_probe_log_err(0, TOO_MANY_ARGS); return -E2BIG; }
ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); if (ret) return ret;
/* setup a probe */ tf = alloc_trace_fprobe(group, event, symbol, argc, is_return, is_tracepoint); if (IS_ERR(tf)) { ret = PTR_ERR(tf); /* This must return -ENOMEM, else there is a bug */ WARN_ON_ONCE(ret != -ENOMEM); return ret; }
/* parse arguments */ for (i = 0; i < argc; i++) { trace_probe_log_set_index(i + 2); ctx->offset = 0; ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], ctx); if (ret) return ret; /* This can be -ENOMEM */ }
if (is_return && tf->tp.entry_arg) { tf->fp.entry_handler = trace_fprobe_entry_handler; tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp); if (ALIGN(tf->fp.entry_data_size, sizeof(long)) > MAX_FPROBE_DATA_SIZE) { trace_probe_log_set_index(2); trace_probe_log_err(0, TOO_MANY_EARGS); return -E2BIG; } }
ret = traceprobe_set_print_fmt(&tf->tp, is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL); if (ret < 0) return ret;
ret = register_trace_fprobe_event(tf); if (ret) { trace_probe_log_set_index(1); if (ret == -EILSEQ) trace_probe_log_err(0, BAD_INSN_BNDRY); else if (ret == -ENOENT) trace_probe_log_err(0, BAD_PROBE_ADDR); else if (ret != -ENOMEM && ret != -EEXIST) trace_probe_log_err(0, FAIL_REG_PROBE); return -EINVAL; }
/* 'tf' is successfully registered. To avoid freeing, assign NULL. */ tf = NULL;
return 0;}
static int trace_fprobe_create_cb(int argc, const char *argv[]){ struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; int ret;
ctx = kzalloc_obj(*ctx); if (!ctx) return -ENOMEM;
ctx->flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE;
trace_probe_log_init("trace_fprobe", argc, argv); ret = trace_fprobe_create_internal(argc, argv, ctx); trace_probe_log_clear(); return ret;}
static int trace_fprobe_create(const char *raw_command){ return trace_probe_create(raw_command, trace_fprobe_create_cb);}
static int trace_fprobe_release(struct dyn_event *ev){ struct trace_fprobe *tf = to_trace_fprobe(ev); int ret = unregister_trace_fprobe(tf);
if (!ret) free_trace_fprobe(tf); return ret;}
static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev){ struct trace_fprobe *tf = to_trace_fprobe(ev); int i;
if (trace_fprobe_is_tracepoint(tf)) seq_putc(m, 't'); else seq_putc(m, 'f'); seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp), trace_probe_name(&tf->tp));
seq_printf(m, " %s%s", trace_fprobe_symbol(tf), trace_fprobe_is_return(tf) ? "%return" : "");
for (i = 0; i < tf->tp.nr_args; i++) seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm); seq_putc(m, '\n');
return 0;}
/* * Enable trace_probe * if the file is NULL, enable "perf" handler, or enable "trace" handler. */static int enable_trace_fprobe(struct trace_event_call *call, struct trace_event_file *file){ struct trace_probe *tp; struct trace_fprobe *tf; bool enabled; int ret = 0;
tp = trace_probe_primary_from_call(call); if (WARN_ON_ONCE(!tp)) return -ENODEV; enabled = trace_probe_is_enabled(tp);
/* This also changes "enabled" state */ if (file) { ret = trace_probe_add_file(tp, file); if (ret) return ret; } else trace_probe_set_flag(tp, TP_FLAG_PROFILE);
if (!enabled) { list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { ret = __register_trace_fprobe(tf); if (ret < 0) return ret; } }
return 0;}
/* * Disable trace_probe * if the file is NULL, disable "perf" handler, or disable "trace" handler. */static int disable_trace_fprobe(struct trace_event_call *call, struct trace_event_file *file){ struct trace_fprobe *tf; struct trace_probe *tp;
tp = trace_probe_primary_from_call(call); if (WARN_ON_ONCE(!tp)) return -ENODEV;
if (file) { if (!trace_probe_get_file_link(tp, file)) return -ENOENT; if (!trace_probe_has_single_file(tp)) goto out; trace_probe_clear_flag(tp, TP_FLAG_TRACE); } else trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
if (!trace_probe_is_enabled(tp)) { list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { unregister_fprobe(&tf->fp); if (tf->tuser) { tracepoint_user_put(tf->tuser); tf->tuser = NULL; } } }
out: if (file) /* * Synchronization is done in below function. For perf event, * file == NULL and perf_trace_event_unreg() calls * tracepoint_synchronize_unregister() to ensure synchronize * event. We don't need to care about it. */ trace_probe_remove_file(tp, file);
return 0;}
/* * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. */static int fprobe_register(struct trace_event_call *event, enum trace_reg type, void *data){ struct trace_event_file *file = data;
switch (type) { case TRACE_REG_REGISTER: return enable_trace_fprobe(event, file); case TRACE_REG_UNREGISTER: return disable_trace_fprobe(event, file);
#ifdef CONFIG_PERF_EVENTS case TRACE_REG_PERF_REGISTER: return enable_trace_fprobe(event, NULL); case TRACE_REG_PERF_UNREGISTER: return disable_trace_fprobe(event, NULL); case TRACE_REG_PERF_OPEN: case TRACE_REG_PERF_CLOSE: case TRACE_REG_PERF_ADD: case TRACE_REG_PERF_DEL: return 0;#endif } return 0;}
/* * Register dynevent at core_initcall. This allows kernel to setup fprobe * events in postcore_initcall without tracefs. */static __init int init_fprobe_trace_early(void){ int ret;
ret = dyn_event_register(&trace_fprobe_ops); if (ret) return ret;
#ifdef CONFIG_MODULES ret = register_tracepoint_module_notifier(&tracepoint_module_nb); if (ret) return ret; ret = register_module_notifier(&tprobe_event_module_nb); if (ret) return ret;#endif
return 0;}core_initcall(init_fprobe_trace_early);