Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643// SPDX-License-Identifier: GPL-2.0/* * C2C function model - function-level cacheline sharing analysis * * Displays a 3-level hierarchy showing which functions share cachelines: * Level 1: Read-side functions sorted by Cycles % (estimated load cycles) * Level 2: Functions sampled writing the shared lines read by level 1 * Level 3: The specific cachelines where the two functions contend * * Builds the hierarchy from the existing cacheline histograms * (c2c_hist_entry->hists), reusing the shared c2c data structures. */
#include <errno.h>#include <inttypes.h>#include <stdlib.h>#include <string.h>#include <tools/libc_compat.h> /* reallocarray */#include <linux/list.h>#include <linux/rbtree.h>#include <linux/zalloc.h>
#include "addr_location.h"#include "c2c.h"#include "cacheline.h"#include "debug.h"#include "dso.h"#include "hist.h"#include "map.h"#include "mem-events.h"#include "mem-info.h"#include "sort.h"#include "symbol.h"#include "thread.h"
struct c2c_function_model { struct c2c_hists function_hists; /* Total estimated cycles across all level-1 entries. */ u64 total_cycles; /* Source cacheline histograms; not owned here. */ struct c2c_hists *cl_hists; /* --coalesce field list, used to require iaddr. */ const char *cl_sort; /* Do not cap long symbol names. */ bool symbol_full;};
static struct c2c_function_model c2c_ext;
static inline u64 c2c_hitm_count(const struct c2c_stats *stats){ return stats->tot_hitm;}
static int64_t c2c_function_cmp(const struct map_symbol *left, const struct map_symbol *right){ const struct dso *left_dso = left->map ? map__dso(left->map) : NULL; const struct dso *right_dso = right->map ? map__dso(right->map) : NULL; int ret;
if (!left_dso || !right_dso) { if (left_dso != right_dso) return left_dso ? 1 : -1; } else { /* * Use the same DSO name as _sort__dso_cmp() (short name unless * verbose), so this matches the DSO comparison the level-1 * entries are deduplicated by; otherwise same-basename DSOs * could be split or merged inconsistently across levels. */ const char *left_name = verbose > 0 ? dso__long_name(left_dso) : dso__short_name(left_dso); const char *right_name = verbose > 0 ? dso__long_name(right_dso) : dso__short_name(right_dso);
ret = strcmp(left_name, right_name); if (ret) return ret; }
return _sort__sym_cmp(left->sym, right->sym);}
static inline u64 hist_entry__iaddr(struct hist_entry *he){ if (he->mem_info) return mem_info__iaddr(he->mem_info)->addr; return he->ip;}
/* * Hierarchy levels (by depth): L1 = read-side function, L2 = the writing * function it contends with, L3 = the specific shared cacheline. */static inline bool hist_entry__is_cacheline(struct hist_entry *he){ return he->parent_he && he->parent_he->parent_he; /* level 3: cacheline */}
/* Spaces of indent per hierarchy level, like the normal report view. */#define C2C_FUNC_INDENT 2
/* Width of the folded-sign prefix ("%c ") each identity cell emits. */#define C2C_FUNC_FOLD_WIDTH 2
/* * Write he->depth levels of leading indentation into @buf, so lower-level * entries are visually nested under their parent. Returns bytes written. */static int hist_entry__indent(struct hist_entry *he, char *buf, size_t size){ int indent = he->depth * C2C_FUNC_INDENT;
if (indent <= 0 || (size_t)indent >= size) return 0;
return scnprintf(buf, size, "%*s", indent, "");}
static int symbol_width(struct hists *hists, struct sort_entry *se){ int width = hists__col_len(hists, se->se_width_idx);
/* * Cap long symbol names as the cacheline view does. The stored column * length is grown up front to fit the deepest, longest identity cell * (including a level-3 cacheline address), so this cap never shrinks the * column below what the cacheline address needs. */ if (!c2c_ext.symbol_full && width > SYMBOL_WIDTH) width = SYMBOL_WIDTH;
return width;}
static struct c2c_dimension dim_symbol_view;
/* * c2c_width - Calculate width for a C2C column in function view */static int c2c_width(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp __maybe_unused, struct hists *hists){ struct c2c_fmt *c2c_fmt; struct c2c_dimension *dim;
c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); dim = c2c_fmt->dim;
if (dim == &dim_symbol_view) return symbol_width(hists, dim->se);
return dim->se ? hists__col_len(hists, dim->se->se_width_idx) : dim->width;}
static int c2c_header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hists *hists, int line, int *span){ struct c2c_fmt *c2c_fmt; struct c2c_dimension *dim; const char *text = NULL; int width = c2c_width(fmt, hpp, hists);
c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); dim = c2c_fmt->dim;
if (dim->se) { text = dim->header.line[line].text; /* Use the last line from sort_entry if not defined. */ if (!text && line == hists->hpp_list->nr_header_lines - 1) text = dim->se->se_header; } else { text = dim->header.line[line].text;
if (span) { if (*span) { (*span)--; return 0; }
*span = dim->header.line[line].span; } }
if (!text) text = "";
return scnprintf(hpp->buf, hpp->size, "%*s", width, text);}
/* * Return the estimated total cycles for a c2c_hist_entry * (rmt_hitm + lcl_hitm + rmt_peer + lcl_peer + other loads). */static u64 c2c_hist_entry__cycles(struct c2c_hist_entry *c2c_he){ struct compute_stats *cs = &c2c_he->cstats; double cycles = 0;
/* * compute_stats() in builtin-c2c.c routes each load sample into exactly * one cstats bucket (rmt_hitm, lcl_hitm, rmt_peer, lcl_peer or plain * load), so each bucket's cycle total is its mean times its own sample * count. Summing the per-bucket totals avoids both dropping peer-snoop * cycles and double counting a sample that carries several data-source * flags (e.g. Arm SPE sets HITM and PEER on the same load), which would * happen if the mean were multiplied by the non-exclusive stats counts. */ cycles += avg_stats(&cs->rmt_hitm) * cs->rmt_hitm.n; cycles += avg_stats(&cs->lcl_hitm) * cs->lcl_hitm.n; cycles += avg_stats(&cs->rmt_peer) * cs->rmt_peer.n; cycles += avg_stats(&cs->lcl_peer) * cs->lcl_peer.n; cycles += avg_stats(&cs->load) * cs->load.n;
return (u64)cycles;}
/* Sum c2c_hist_entry__cycles() across all level-1 entries. */static u64 c2c_ext__total_cycles(void){ struct rb_node *nd; u64 total = 0;
for (nd = rb_first_cached(&c2c_ext.function_hists.hists.entries); nd; nd = rb_next(nd)) { struct c2c_hist_entry *c2c_he = rb_entry(nd, struct c2c_hist_entry, he.rb_node);
total += c2c_hist_entry__cycles(c2c_he); } return total;}
/* * Store count shown in the column: a level-3 cacheline leaf shows its parent * level-2 writer's stores on that line, not all stores on the line. A level-2 * writer shows the sum across its level-3 cachelines. A level-1 reader shows * the sum across all included writers on the cachelines it reads; this is not * the reader function's own store count and is not additive across readers. */static u64 hist_entry__displayed_stores(struct hist_entry *he){ struct c2c_hist_entry *c2c_he = container_of(he, struct c2c_hist_entry, he); struct rb_node *nd; u64 stores = 0;
/* Level-2/3 entries already aggregate the stores they represent. */ if (he->parent_he) return c2c_he->stats.store;
for (nd = rb_first_cached(&he->hroot_out); nd; nd = rb_next(nd)) { struct c2c_hist_entry *child_c2c = rb_entry(nd, struct c2c_hist_entry, he.rb_node); stores += child_c2c->stats.store; } return stores;}
static inttotal_stores_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he){ int width = c2c_width(fmt, hpp, he->hists); u64 total = hist_entry__displayed_stores(he);
return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, total);}
/* * symbol_view_entry - Render the unified, indented identity column. * * All three levels share this single column so the hierarchy reads top-down * with progressive indentation, like the normal report hierarchy view. It is * a function-centric view with no dedicated code-address column. Verbose * function rows can still include a representative address: * L1 read-side function: "- [k] cpupri_set" * L2 writing function: " - [k] pull_rt_task" * L3 shared cacheline: " 0xff2d0082809da080" */static intsymbol_view_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he){ int width = c2c_width(fmt, hpp, he->hists); int text_width; int ret; char folded_sign;
ret = hist_entry__indent(he, hpp->buf, hpp->size);
folded_sign = he->has_children ? (he->unfolded ? '-' : '+') : ' '; ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%c ", folded_sign);
text_width = width - ret; if (text_width <= 0) return ret;
if (hist_entry__is_cacheline(he)) { /* Level 3: the shared cacheline address. */ u64 addr = he->mem_info ? cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl) : 0; char symbuf[32];
scnprintf(symbuf, sizeof(symbuf), "0x%" PRIx64, addr); ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%-*.*s", text_width, text_width, symbuf); } else { /* Level 1 and level 2 are both functions. */ size_t cell_size; int len;
if ((size_t)ret >= hpp->size) return ret;
cell_size = min_t(size_t, hpp->size - ret, (size_t)text_width + 1); len = sort_sym.se_snprintf(he, hpp->buf + ret, cell_size, text_width); /* * se_snprintf() accumulates repsep_snprintf() calls, which cap * their return at the remaining size - 1 rather than reporting * what the format would have needed, so len stays below * cell_size. Clamp anyway so ret cannot leave hpp->buf. */ if (len < 0) len = 0; else len = min_t(size_t, len, cell_size - 1);
ret += len; if (len < text_width) ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%*s", text_width - len, ""); }
return ret;}
/* * cycles_percent_entry - Render cycles percentage column */static intcycles_percent_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he){ struct c2c_hist_entry *c2c_he; int width = c2c_width(fmt, hpp, he->hists); u64 fn_cycles, total_cycles; char folded_sign; double pct; int ret, pct_width;
/* Hide Cycles Percent for child functions and cachelines. */ if (he->parent_he) return scnprintf(hpp->buf, hpp->size, "%*s", width, "");
c2c_he = container_of(he, struct c2c_hist_entry, he); fn_cycles = c2c_hist_entry__cycles(c2c_he); /* Populated by c2c_function__build() once the L1 tree is built. */ total_cycles = c2c_ext.total_cycles; pct = total_cycles > 0 ? (double)fn_cycles / total_cycles * 100.0 : 0.0;
/* Add folded sign only for level-1 entries */ folded_sign = he->has_children ? (he->unfolded ? '-' : '+') : ' '; ret = scnprintf(hpp->buf, hpp->size, "%c ", folded_sign);
pct_width = width - ret; if (pct_width <= 0) return ret; ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%*.2f%%", pct_width - 1, pct); return ret;}
/* * cycles_percent_cmp - Comparison function for cycles percentage sorting */static int64_tcycles_percent_cmp(struct perf_hpp_fmt *fmt __maybe_unused, struct hist_entry *left, struct hist_entry *right){ struct c2c_hist_entry *c2c_left = container_of(left, struct c2c_hist_entry, he); struct c2c_hist_entry *c2c_right = container_of(right, struct c2c_hist_entry, he); u64 cycles_left, cycles_right;
/* Cycles Percent is only shown for level-1 entries; others compare equal. */ if (left->parent_he || right->parent_he) return 0;
cycles_left = c2c_hist_entry__cycles(c2c_left); cycles_right = c2c_hist_entry__cycles(c2c_right);
return (cycles_left > cycles_right) - (cycles_left < cycles_right);}
/* * total_stores_cmp - Comparison function for total stores sorting */static int64_ttotal_stores_cmp(struct perf_hpp_fmt *fmt __maybe_unused, struct hist_entry *left, struct hist_entry *right){ u64 left_store = hist_entry__displayed_stores(left); u64 right_store = hist_entry__displayed_stores(right);
return (left_store > right_store) - (left_store < right_store);}
/* * Function view dimensions */static struct c2c_dimension dim_cycles_percent = { .header = HEADER_BOTH("Cycles", "%"), .name = "cycles_percent", .cmp = cycles_percent_cmp, .entry = cycles_percent_entry, .width = 9,};
static struct c2c_dimension dim_total_stores = { .header = HEADER_BOTH("Store", "count"), .name = "total_stores", .cmp = total_stores_cmp, .entry = total_stores_entry, .width = 7,};
static struct c2c_dimension dim_symbol_view = { .header = HEADER_LOW("Function / Contending function / Cacheline"), .name = "symbol_view", .se = &sort_sym, .entry = symbol_view_entry, .width = SYMBOL_WIDTH,};
static struct c2c_dimension *function_view_dimensions[] = { &dim_cycles_percent, &dim_total_stores, &dim_symbol_view, NULL,};
static struct c2c_dimension *get_function_dimension(const char *name){ unsigned int i;
for (i = 0; function_view_dimensions[i]; i++) { struct c2c_dimension *dim = function_view_dimensions[i];
if (!strcmp(dim->name, name)) return dim; }
return NULL;}
/* Wrappers so sort_entry-backed dimensions sort/collapse via their se. */static int64_t c2c_se_cmp(struct perf_hpp_fmt *fmt, struct hist_entry *a, struct hist_entry *b){ struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); struct c2c_dimension *dim = c2c_fmt->dim;
return dim->se->se_cmp(a, b);}
static int64_t c2c_se_collapse(struct perf_hpp_fmt *fmt, struct hist_entry *a, struct hist_entry *b){ struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); struct c2c_dimension *dim = c2c_fmt->dim; int64_t (*collapse_fn)(struct hist_entry *a, struct hist_entry *b);
collapse_fn = dim->se->se_collapse ?: dim->se->se_cmp; return collapse_fn(a, b);}
static int64_t c2c_se_sort(struct perf_hpp_fmt *fmt, struct hist_entry *a, struct hist_entry *b){ struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); struct c2c_dimension *dim = c2c_fmt->dim; int64_t (*sort_fn)(struct hist_entry *a, struct hist_entry *b);
sort_fn = dim->se->se_sort ?: dim->se->se_cmp; return sort_fn(a, b);}
/* * Build the c2c_fmt for @name. Returns: * 0 and *fmtp set on success; * -ENOENT and *fmtp = NULL if @name is not a function-view dimension; * -ENOMEM if allocation failed (distinct from -ENOENT so * the caller does not misreport it as an * "invalid field"). */static int get_function_format(const char *name, struct c2c_fmt **fmtp){ struct c2c_dimension *dim = get_function_dimension(name); struct c2c_fmt *c2c_fmt; struct perf_hpp_fmt *fmt;
*fmtp = NULL;
if (!dim) return -ENOENT;
c2c_fmt = zalloc(sizeof(*c2c_fmt)); if (!c2c_fmt) return -ENOMEM;
fmt = &c2c_fmt->fmt;
c2c_fmt->dim = dim; INIT_LIST_HEAD(&fmt->list); INIT_LIST_HEAD(&fmt->sort_list);
fmt->cmp = dim->se ? c2c_se_cmp : dim->cmp; fmt->sort = dim->se ? c2c_se_sort : dim->cmp; fmt->color = dim->color; fmt->entry = dim->entry; fmt->header = c2c_header; fmt->width = c2c_width; fmt->collapse = dim->se ? c2c_se_collapse : dim->cmp; fmt->equal = c2c_fmt_equal; fmt->free = c2c_fmt_free;
*fmtp = c2c_fmt; return 0;}
static intc2c_function_hists__init_output(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env __maybe_unused){ struct c2c_fmt *c2c_fmt; int ret;
ret = get_function_format(name, &c2c_fmt); if (ret == -ENOMEM) return ret; /* The function view only accepts its own dimensions. */ if (ret == -ENOENT) return -EINVAL;
/* * Mark symbol-backed columns so hists__has(hists, sym) is correct. * Only dim_symbol_view carries a sort_entry (.se); the function * view's field strings are fixed and always include symbol_view, so * this single check is sufficient (unlike the user-configurable * cacheline view, which must also test dim_iaddr). */ if (c2c_fmt->dim->se == &sort_sym) hpp_list->sym = 1;
perf_hpp_list__column_register(hpp_list, &c2c_fmt->fmt); return 0;}
static intc2c_function_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env __maybe_unused){ struct c2c_fmt *c2c_fmt; int ret;
ret = get_function_format(name, &c2c_fmt); if (ret == -ENOMEM) return ret; /* The function view only accepts its own dimensions. */ if (ret == -ENOENT) return -EINVAL;
/* Mark symbol-backed sort keys so hists__has(hists, sym) is correct. */ if (c2c_fmt->dim->se == &sort_sym) hpp_list->sym = 1;
perf_hpp_list__register_sort_field(hpp_list, &c2c_fmt->fmt); return 0;}
typedef int (*hpp_list_add_fn)(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env);
static int function_hpp_list__add_tokens(struct perf_hpp_list *hpp_list, char *list, struct perf_env *env, hpp_list_add_fn add){ char *tok, *tmp; int ret;
if (!list) return 0;
for (tok = strtok_r(list, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { ret = add(hpp_list, tok, env); if (ret) { if (ret == -EINVAL || ret == -ESRCH) pr_err("Invalid c2c function-view field: %s\n", tok); return ret; } } return 0;}
/* * Append the function view's sort keys to its own output fields, mirroring * perf_hpp__setup_output_field() but on the local @list. The shared helper * registers onto the global perf_hpp_list, which would leave this local list * without output columns, so the function view keeps its own copy here. */static void c2c_function_hists__setup_output_field(struct perf_hpp_list *list){ struct perf_hpp_fmt *fmt;
perf_hpp_list__for_each_sort_list(list, fmt) { struct perf_hpp_fmt *pos;
if (!fmt->entry && !fmt->color) continue;
perf_hpp_list__for_each_format(list, pos) { if (c2c_fmt_equal(fmt, pos)) goto next; }
perf_hpp_list__column_register(list, fmt);next: continue; }}
static intfunction_hpp_list__parse(struct perf_hpp_list *hpp_list, const char *output_str, const char *sort_str, struct perf_env *env){ char *output = output_str ? strdup(output_str) : NULL; char *sort = sort_str ? strdup(sort_str) : NULL; int ret = 0;
if ((output_str && !output) || (sort_str && !sort)) { ret = -ENOMEM; goto out; }
ret = function_hpp_list__add_tokens(hpp_list, output, env, c2c_function_hists__init_output); if (ret) goto out;
ret = function_hpp_list__add_tokens(hpp_list, sort, env, c2c_function_hists__init_sort); if (ret) goto out;
c2c_function_hists__setup_output_field(hpp_list);out: if (ret) perf_hpp__reset_output_field(hpp_list); free(output); free(sort); return ret;}
static intc2c_function_hists__init(struct c2c_hists *hists, const char *sort, int nr_header_lines, struct perf_env *env){ __hists__init(&hists->hists, &hists->list);
perf_hpp_list__init(&hists->list);
hists->list.nr_header_lines = nr_header_lines;
return function_hpp_list__parse(&hists->list, /*output=*/NULL, sort, env);}
static intc2c_function_hists__reinit(struct c2c_hists *c2c_hists, const char *output, const char *sort, struct perf_env *env){ int nr_header_lines = c2c_hists->list.nr_header_lines;
perf_hpp__reset_output_field(&c2c_hists->list);
/* Clear stale state flags so a different output/sort set starts fresh. */ c2c_hists->list.need_collapse = 0; c2c_hists->list.parent = 0; c2c_hists->list.sym = 0; c2c_hists->list.dso = 0; c2c_hists->list.socket = 0; c2c_hists->list.thread = 0; c2c_hists->list.comm = 0; c2c_hists->list.comm_nodigit = 0; c2c_hists->list.nr_header_lines = nr_header_lines;
return function_hpp_list__parse(&c2c_hists->list, output, sort, env);}
/* Welford online merge of two "stats" (from util/stat.h) accumulators. */static void c2c_stats_merge(struct stats *dest, const struct stats *src){ double delta;
if (src->n == 0) return;
if (dest->n == 0) { *dest = *src; return; }
delta = src->mean - dest->mean; dest->M2 += src->M2 + delta * delta * dest->n * src->n / (dest->n + src->n); dest->mean = (dest->mean * dest->n + src->mean * src->n) / (dest->n + src->n); dest->n += src->n;
/* Update min/max */ if (src->max > dest->max) dest->max = src->max; if (src->min < dest->min) dest->min = src->min;}
/* Merge compute_stats during function aggregation. */static void c2c_add_cstats(struct compute_stats *dest, const struct compute_stats *src){ c2c_stats_merge(&dest->rmt_hitm, &src->rmt_hitm); c2c_stats_merge(&dest->lcl_hitm, &src->lcl_hitm); c2c_stats_merge(&dest->rmt_peer, &src->rmt_peer); c2c_stats_merge(&dest->lcl_peer, &src->lcl_peer); c2c_stats_merge(&dest->load, &src->load);}
static bool hist_entry__add_c2c_stats(struct hist_entry *he, const struct c2c_stats *stats){ u64 nr_events = c2c_hitm_count(stats) + stats->rmt_peer + stats->lcl_peer; u64 weight1 = c2c_hitm_count(stats);
/* * Allocate before touching he->stat, so a failure here leaves the * entry unmodified and the caller can bail out without having * half-updated the statistics. */ if (symbol_conf.cumulate_callchain && !he->stat_acc) { he->stat_acc = calloc(1, sizeof(struct he_stat)); if (!he->stat_acc) return false; }
he->stat.nr_events += nr_events; he->stat.period += nr_events; he->stat.weight1 += weight1;
if (!symbol_conf.cumulate_callchain) return true;
he->stat_acc->nr_events += nr_events; he->stat_acc->period += nr_events; he->stat_acc->weight1 += weight1;
return true;}
static void c2c_he__free_hierarchy(struct hist_entry *he);
/* * Free a function-view histogram entry (hist_entry_ops::free). */static void c2c_function_he_free(void *ptr){ struct hist_entry *he = ptr; struct c2c_hist_entry *c2c_he;
c2c_he = container_of(he, struct c2c_hist_entry, he);
if (c2c_he->hists) { perf_hpp__reset_output_field(&c2c_he->hists->list); hists__delete_all_entries(&c2c_he->hists->hists); zfree(&c2c_he->hists); }
c2c_he__free_hierarchy(he);
free(c2c_he);}
static void c2c_he__free_hierarchy(struct hist_entry *he){ struct rb_node *nd; struct hist_entry *child_he;
/* * A leaf entry stores its callchains in the sorted_chain member, which * shares a union with the hroot_in/hroot_out child trees, so its * hroot_out is not a valid subtree to walk. Leaf entries never have a * child hierarchy here, so stop before touching hroot_out. */ if (he->leaf) return;
if (RB_EMPTY_ROOT(&he->hroot_out.rb_root)) return;
nd = rb_first_cached(&he->hroot_out); while (nd) { struct rb_node *next = rb_next(nd);
child_he = rb_entry(nd, struct hist_entry, rb_node); rb_erase_cached(&child_he->rb_node, &he->hroot_out); hist_entry__delete(child_he);
nd = next; }
/* All children erased; clear the tree (and its cached leftmost). */ he->hroot_out = RB_ROOT_CACHED;}
/* * Drop level-2 writing functions that carry no stores or * no cacheline children. Writers are only added when they store into a shared * line, so this is mainly a safety net. Returns the number of surviving * writers. */static int c2c_he__prune_empty_writers(struct hist_entry *l1_he){ struct rb_node *nd; int surviving = 0;
if (!l1_he->has_children) return 0;
nd = rb_first_cached(&l1_he->hroot_out); while (nd) { struct rb_node *next = rb_next(nd); struct hist_entry *l2_he = rb_entry(nd, struct hist_entry, rb_node);
if (l2_he->has_children && hist_entry__displayed_stores(l2_he) > 0) { surviving++; } else { rb_erase_cached(&l2_he->rb_node, &l1_he->hroot_out); hist_entry__delete(l2_he); } nd = next; }
if (!surviving) { l1_he->hroot_out = RB_ROOT_CACHED; l1_he->has_children = false; l1_he->unfolded = false; } return surviving;}
static void *c2c_function_he_zalloc(size_t size){ struct c2c_hist_entry *c2c_he = zalloc(sizeof(*c2c_he) + size);
if (!c2c_he) return NULL;
init_stats(&c2c_he->cstats.lcl_hitm); init_stats(&c2c_he->cstats.rmt_hitm); init_stats(&c2c_he->cstats.lcl_peer); init_stats(&c2c_he->cstats.rmt_peer); init_stats(&c2c_he->cstats.load);
return &c2c_he->he;}
/* Entry operations for function view */static struct hist_entry_ops c2c_function_entry_ops = { .new = c2c_function_he_zalloc, .free = c2c_function_he_free,};
static struct c2c_hist_entry *c2c_child_entry__alloc(struct hist_entry *parent_he, struct hist_entry *src_he, int depth, u64 ip){ struct c2c_hist_entry *child_c2c; struct hist_entry *child_he;
/* Function-view children never own or display callchains. */ child_he = c2c_function_he_zalloc(0); if (!child_he) return NULL;
child_c2c = container_of(child_he, struct c2c_hist_entry, he); child_he->ops = &c2c_function_entry_ops; map_symbol__copy(&child_he->ms, &src_he->ms);
if (src_he->mem_info) { child_he->mem_info = mem_info__clone(src_he->mem_info); if (!child_he->mem_info) goto out_free; }
child_he->thread = thread__get(src_he->thread); child_he->cpumode = src_he->cpumode; child_he->cpu = src_he->cpu; child_he->socket = src_he->socket; child_he->level = src_he->level; child_he->ip = ip;
child_he->parent_he = parent_he; child_he->depth = depth; child_he->leaf = (depth >= 2); child_he->hists = &c2c_ext.function_hists.hists; child_he->filtered = false; child_he->unfolded = false; child_he->has_children = false; child_he->has_no_entry = false; child_he->nr_rows = 0; child_he->row_offset = 0;
memset(&child_he->stat, 0, sizeof(child_he->stat)); child_he->hroot_in = RB_ROOT_CACHED; child_he->hroot_out = RB_ROOT_CACHED; INIT_LIST_HEAD(&child_he->pairs.node); child_he->hpp_list = &c2c_ext.function_hists.list; if (symbol_conf.cumulate_callchain) { child_he->stat_acc = calloc(1, sizeof(struct he_stat)); if (!child_he->stat_acc) goto out_free; }
return child_c2c;
out_free: hist_entry__delete(child_he); return NULL;}
static voidc2c_child_entry__insert(struct hist_entry *parent_he, struct hist_entry *child_he, struct rb_node **p, struct rb_node *rb_parent, bool leftmost){ rb_link_node(&child_he->rb_node, rb_parent, p); rb_insert_color_cached(&child_he->rb_node, &parent_he->hroot_out, leftmost);
parent_he->has_children = true; parent_he->leaf = false;}
static struct hist_entry *c2c_function_hists__level1_entry(struct symbol *sym, struct hist_entry *detail_he, struct thread *synthetic_thread){ struct addr_location al; struct perf_sample sample = {}; struct mem_info *mi; struct hist_entry *he; /* * Key the level-1 entry by the function, not by a specific code * address: use the symbol start so every instruction address inside * the same function collapses into one entry. This makes level 1 a * true "function view" rather than a per-code-address view. */ u64 sym_start = (sym && detail_he->ms.map) ? map__unmap_ip(detail_he->ms.map, sym->start) : detail_he->ip;
mi = mem_info__new(); if (!mi) return NULL;
mem_info__iaddr(mi)->addr = sym_start; /* mem_info__put() will map_symbol__exit() these, so take refs. */ mem_info__iaddr(mi)->ms.thread = thread__get(detail_he->ms.thread); mem_info__iaddr(mi)->ms.map = map__get(detail_he->ms.map); mem_info__iaddr(mi)->ms.sym = sym; mem_info__daddr(mi)->addr = 0;
addr_location__init(&al); al.thread = thread__get(synthetic_thread); al.map = map__get(detail_he->ms.map); al.sym = sym; al.addr = sym_start; al.level = detail_he->level; al.cpumode = detail_he->cpumode; al.cpu = 0; al.socket = 0; al.filtered = 0; al.latency = 0;
/* * Synthetic sample: period/weight are placeholders only. The real * c2c counters live in c2c_hist_entry::stats and are added via * hist_entry__add_c2c_stats(); no function-view column or sort key * reads he->stat.period/nr_events, so the +1 that __hists__add_entry() * accrues on each dedup hit has no effect on what is displayed. */ sample.period = 1; sample.weight = 1; sample.ip = sym_start; sample.pid = thread__pid(synthetic_thread); sample.tid = thread__tid(synthetic_thread); sample.cpu = 0;
/* Add entry - histogram handles dedup */ he = hists__add_entry_ops(&c2c_ext.function_hists.hists, &c2c_function_entry_ops, &al, NULL, NULL, mi, NULL, &sample, true);
addr_location__exit(&al); mem_info__put(mi);
if (he) he->hpp_list = &c2c_ext.function_hists.list;
return he;}
/* * Level 2: a function that writes a cacheline the level-1 function reads, * keyed by the DSO display name and symbol, consistently with perf's symbol * sort semantics. All code addresses and cachelines for the same writer * function aggregate into one row. */static struct c2c_hist_entry *c2c_function_hists__level2_entry(struct c2c_hist_entry *level1_c2c, struct symbol *sym, struct hist_entry *detail_he){ struct hist_entry *level1_he = &level1_c2c->he; struct rb_node **p = &level1_he->hroot_out.rb_root.rb_node; struct rb_node *parent = NULL; struct c2c_hist_entry *level2_c2c; bool leftmost = true;
while (*p) { struct hist_entry *iter = rb_entry(*p, struct hist_entry, rb_node); struct map_symbol key = detail_he->ms; int64_t cmp;
key.sym = sym; parent = *p; cmp = c2c_function_cmp(&key, &iter->ms);
if (cmp < 0) { p = &parent->rb_left; } else if (cmp > 0) { p = &parent->rb_right; leftmost = false; } else { return container_of(iter, struct c2c_hist_entry, he); } }
/* Key by the function symbol start so all code addresses collapse. */ level2_c2c = c2c_child_entry__alloc(level1_he, detail_he, 1, (sym && detail_he->ms.map) ? map__unmap_ip(detail_he->ms.map, sym->start) : hist_entry__iaddr(detail_he)); if (!level2_c2c) return NULL;
/* Key this level by the looked-up symbol, not detail_he's. */ level2_c2c->he.ms.sym = sym; if (level2_c2c->he.mem_info) mem_info__iaddr(level2_c2c->he.mem_info)->ms.sym = sym;
c2c_child_entry__insert(level1_he, &level2_c2c->he, p, parent, leftmost);
return level2_c2c;}
/* Level 3: one source cacheline where the L1/L2 functions contend. */static struct c2c_hist_entry *c2c_function_hists__level3_entry(struct c2c_hist_entry *level2_c2c, struct c2c_hist_entry *cacheline_src_he){ struct hist_entry *level2_he = &level2_c2c->he; struct rb_node **p = &level2_he->hroot_out.rb_root.rb_node; struct rb_node *parent = NULL; struct c2c_hist_entry *level3_c2c; bool leftmost = true;
while (*p) { struct c2c_hist_entry *iter_c2c = rb_entry(*p, struct c2c_hist_entry, he.rb_node);
parent = *p; if (cacheline_src_he->cacheline_idx < iter_c2c->cacheline_idx) { p = &parent->rb_left; } else if (cacheline_src_he->cacheline_idx > iter_c2c->cacheline_idx) { p = &parent->rb_right; leftmost = false; } else { return iter_c2c; } }
level3_c2c = c2c_child_entry__alloc(level2_he, &cacheline_src_he->he, 2, hist_entry__iaddr(&cacheline_src_he->he)); if (!level3_c2c) return NULL; level3_c2c->cacheline_idx = cacheline_src_he->cacheline_idx;
c2c_child_entry__insert(level2_he, &level3_c2c->he, p, parent, leftmost);
return level3_c2c;}
struct hist_entry *c2c_function__find_cacheline(struct hist_entry *he_selection){ struct c2c_hist_entry *c2c_he; struct rb_node *nd;
if (!c2c_ext.cl_hists || !he_selection || !he_selection->parent_he || !he_selection->parent_he->parent_he) return NULL;
c2c_he = container_of(he_selection, struct c2c_hist_entry, he);
for (nd = rb_first_cached(&c2c_ext.cl_hists->hists.entries); nd; nd = rb_next(nd)) { struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); struct c2c_hist_entry *cacheline_he;
if (he->filtered) continue;
cacheline_he = container_of(he, struct c2c_hist_entry, he); if (cacheline_he->hists && cacheline_he->cacheline_idx == c2c_he->cacheline_idx) return he; }
return NULL;}
/* * Re-sort child entries of @parent_he by total store count, descending. */static void c2c_he__resort_by_stores(struct hist_entry *parent_he){ struct rb_root_cached new_root = RB_ROOT_CACHED; struct rb_node *nd;
if (!parent_he->has_children) return;
/* Extract all nodes and re-insert sorted by displayed store count */ while ((nd = rb_first_cached(&parent_he->hroot_out))) { struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); u64 he_store = hist_entry__displayed_stores(he); struct rb_node **p = &new_root.rb_root.rb_node; struct rb_node *parent = NULL; bool leftmost = true; int cmp;
/* Remove from current tree */ rb_erase_cached(&he->rb_node, &parent_he->hroot_out);
/* * Insert sorted by store count, descending. Use the displayed * store count so a level-1 function and level-2 writer (whose own * stats.store is 0 / partial) sort by the aggregated write traffic * beneath them, not by their own store field. */ while (*p) { struct hist_entry *iter = rb_entry(*p, struct hist_entry, rb_node); u64 iter_store = hist_entry__displayed_stores(iter);
parent = *p; if (he_store != iter_store) { cmp = he_store > iter_store ? -1 : 1; } else { /* Stable tie-break: instruction address, name, then cacheline. */ u64 a = hist_entry__iaddr(he), b = hist_entry__iaddr(iter);
if (a != b) cmp = a < b ? -1 : 1; else if (he->ms.sym && iter->ms.sym) cmp = strcmp(he->ms.sym->name, iter->ms.sym->name); else cmp = (iter->ms.sym ? 1 : 0) - (he->ms.sym ? 1 : 0);
if (!cmp) { struct c2c_hist_entry *he_c2c; struct c2c_hist_entry *iter_c2c;
he_c2c = container_of(he, struct c2c_hist_entry, he); iter_c2c = container_of(iter, struct c2c_hist_entry, he); if (he_c2c->cacheline_idx != iter_c2c->cacheline_idx) cmp = he_c2c->cacheline_idx < iter_c2c->cacheline_idx ? -1 : 1; } }
if (cmp < 0) { p = &parent->rb_left; } else { p = &parent->rb_right; leftmost = false; } }
rb_link_node(&he->rb_node, parent, p); rb_insert_color_cached(&he->rb_node, &new_root, leftmost); }
parent_he->hroot_out = new_root;}
/* Initial per-cacheline capacity for the seen[] set; grown on demand. */#define DEFAULT_SYMBOLS_PER_CL 64
struct function_seen { struct map_symbol ms;};
static bool function_seen__find(const struct function_seen *seen, int nr, const struct map_symbol *ms){ int i;
for (i = 0; i < nr; i++) { if (!c2c_function_cmp(&seen[i].ms, ms)) return true; } return false;}
/* Aggregate stats from the cacheline-side entry @c2c_b into level 2/3 @dst. */static bool c2c_he__add_sharing(struct c2c_hist_entry *dst, struct c2c_hist_entry *src){ /* Do the fallible update first so a failure leaves dst unmodified. */ if (!hist_entry__add_c2c_stats(&dst->he, &src->stats)) return false;
c2c_add_stats(&dst->stats, &src->stats); c2c_add_cstats(&dst->cstats, &src->cstats); return true;}
/* * Process one cacheline: for every function reading it, create/update its * level-1 function entry, then for each function that writes the line * add it as a level-2 writer and add this cacheline as a level-3 child. */static int c2c_function__process_cl(struct c2c_hist_entry *cacheline_he, struct thread *synthetic_thread){ struct rb_node *nd_a, *nd_b; struct function_seen *seen = NULL; int nr_seen = 0, nr_alloc = 0; int ret = 0;
for (nd_a = rb_first_cached(&cacheline_he->hists->hists.entries); nd_a; nd_a = rb_next(nd_a)) { struct hist_entry *he_a = rb_entry(nd_a, struct hist_entry, rb_node); struct c2c_hist_entry *c2c_a; struct hist_entry *level1_he; struct c2c_hist_entry *level1_c2c;
if (!he_a->ms.sym || he_a->filtered) continue;
c2c_a = container_of(he_a, struct c2c_hist_entry, he); if (c2c_a->stats.load == 0) continue;
level1_he = c2c_function_hists__level1_entry(he_a->ms.sym, he_a, synthetic_thread); if (!level1_he) { ret = -ENOMEM; goto out; }
level1_c2c = container_of(level1_he, struct c2c_hist_entry, he);
/* * Aggregate every source entry into its level-1 function parent. * level1_he is keyed by symbol, so all code addresses inside the * same function collapse into one parent. When the cacheline view * splits a function into siblings (different code addresses, or * --coalesce pid/tid/dso), each sibling holds a DISJOINT slice of the * traffic, so summing them here is correct accumulation, not * double counting. The seen[] set below therefore guards only the * subtree build (to avoid building a function's level-2/3 subtree * twice for the same cacheline), never this L1 update. Update * he->stat first; on failure leave the aggregates untouched. */ if (!hist_entry__add_c2c_stats(level1_he, &c2c_a->stats)) { ret = -ENOMEM; goto out; } c2c_add_stats(&level1_c2c->stats, &c2c_a->stats); c2c_add_cstats(&level1_c2c->cstats, &c2c_a->cstats); c2c_add_stats(&c2c_ext.function_hists.stats, &c2c_a->stats);
/* Build this function's subtree for this cacheline only once. */ if (function_seen__find(seen, nr_seen, &he_a->ms)) continue;
if (nr_seen == nr_alloc) { struct function_seen *tmp; int new_alloc = nr_alloc ? nr_alloc * 2 : DEFAULT_SYMBOLS_PER_CL;
tmp = reallocarray(seen, new_alloc, sizeof(*seen)); if (!tmp) { ret = -ENOMEM; goto out; } seen = tmp; nr_alloc = new_alloc; } seen[nr_seen].ms = he_a->ms; nr_seen++;
for (nd_b = rb_first_cached(&cacheline_he->hists->hists.entries); nd_b; nd_b = rb_next(nd_b)) { struct hist_entry *he_b = rb_entry(nd_b, struct hist_entry, rb_node); struct c2c_hist_entry *c2c_b, *level2_c2c, *level3_c2c;
if (!he_b->ms.sym || he_b->filtered) continue;
c2c_b = container_of(he_b, struct c2c_hist_entry, he);
/* * The level-1 function contributes read-side load weight for this * cacheline. Associate it with functions sampled storing to the * same line. * The writer can be the same function; after detail coalescing and * function-level grouping there is not enough identity to attribute * that case to a specific thread. * Only writers are contending functions, so keep the ones * that actually store into the line. */ if (c2c_b->stats.store == 0) continue;
/* Level 2: the writing function (aggregated across cachelines). */ level2_c2c = c2c_function_hists__level2_entry(level1_c2c, he_b->ms.sym, he_b); if (!level2_c2c || !c2c_he__add_sharing(level2_c2c, c2c_b)) { ret = -ENOMEM; goto out; }
/* Level 3: the specific cacheline they contend over. */ level3_c2c = c2c_function_hists__level3_entry(level2_c2c, cacheline_he); if (!level3_c2c || !c2c_he__add_sharing(level3_c2c, c2c_b)) { ret = -ENOMEM; goto out; } } }
out: free(seen); return ret;}
/* * Remove a level-1 function that has no contended cachelines left. It is a * normal (owned) hist_entry in function_hists, so mirror hists__delete_entry() * for the no-collapse case: unlink from both trees, fix the counters, then * delete. Its hroot_out is already empty after pruning. */static void c2c_function__drop_level1(struct hist_entry *he){ struct hists *hists = &c2c_ext.function_hists.hists;
rb_erase_cached(&he->rb_node_in, hists->entries_in); rb_erase_cached(&he->rb_node, &hists->entries);
--hists->nr_entries; if (!he->filtered) --hists->nr_non_filtered_entries;
hist_entry__delete(he);}
/* Length of the identity text (symbol name or cacheline address) at @he. */static int c2c_function__ident_len(struct hist_entry *he){ char buf[512]; char *symbuf; size_t size; int len;
if (hist_entry__is_cacheline(he)) { u64 addr = he->mem_info ? cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl) : 0;
return scnprintf(buf, sizeof(buf), "0x%" PRIx64, addr); }
if (!he->ms.sym) return 0;
/* * Match symbol_view_entry(): sort_sym adds the cpumode prefix and, in * verbose mode, the address and DSO origin before the symbol name. */ size = strlen(he->ms.sym->name) + 64; symbuf = malloc(size); if (!symbuf) return size - 1;
len = sort_sym.se_snprintf(he, symbuf, size, size - 1); free(symbuf); return len;}
/* * Grow the symbol column so the deepest, longest identity cell fits. The * generic hists__calc_col_len() only measures the top-level (L1) entries; the * hand-linked L2 writers and L3 cacheline addresses live in hroot_out and are * never measured, so with a short L1 name the indented L2/L3 text would be * truncated. Account for the per-level indent and the folded-sign prefix. */static void c2c_function__update_symbol_width(struct hist_entry *he){ struct hists *hists = &c2c_ext.function_hists.hists; int need = he->depth * C2C_FUNC_INDENT + C2C_FUNC_FOLD_WIDTH + c2c_function__ident_len(he);
if (need > hists__col_len(hists, HISTC_SYMBOL)) hists__set_col_len(hists, HISTC_SYMBOL, need);}
/* * Prune writers with no stores, drop functions left with no contending * writer, sort the survivors by store count, then compute the global total. */static void c2c_function__finalize(void){ struct rb_node *nd_l1;
nd_l1 = rb_first_cached(&c2c_ext.function_hists.hists.entries); while (nd_l1) { struct hist_entry *he_l1 = rb_entry(nd_l1, struct hist_entry, rb_node); struct rb_node *next_l1 = rb_next(nd_l1); struct rb_node *nd_l2;
/* Drop writers with no stores before sorting. */ if (!he_l1->has_children || !c2c_he__prune_empty_writers(he_l1)) { /* No contending writer: this function is not shared. */ c2c_function__drop_level1(he_l1); nd_l1 = next_l1; continue; }
c2c_he__resort_by_stores(he_l1); c2c_function__update_symbol_width(he_l1);
for (nd_l2 = rb_first_cached(&he_l1->hroot_out); nd_l2; nd_l2 = rb_next(nd_l2)) { struct hist_entry *he_l2 = rb_entry(nd_l2, struct hist_entry, rb_node); struct rb_node *nd_l3;
c2c_function__update_symbol_width(he_l2);
if (he_l2->has_children) c2c_he__resort_by_stores(he_l2);
for (nd_l3 = rb_first_cached(&he_l2->hroot_out); nd_l3; nd_l3 = rb_next(nd_l3)) { struct hist_entry *he_l3 = rb_entry(nd_l3, struct hist_entry, rb_node);
c2c_function__update_symbol_width(he_l3); } }
nd_l1 = next_l1; }
/* * Compute the Cycles % denominator from the surviving level-1 entries * after pruning, so the column shows each function's share of the * functions retained in this table -- not of the whole recording. See * the Cycles % description in perf-c2c.txt. */ c2c_ext.total_cycles = c2c_ext__total_cycles();}
/* * Release all per-visit function-view state. Keep the hists object itself * initialized so its mutex is initialized exactly once across TAB re-entry. */void c2c_function__reset(void){ bool saved_use_callchain = symbol_conf.use_callchain;
/* * Function-view entries never carry callchains. Keep their generic * destructor independent of the caller's current callchain setting. */ symbol_conf.use_callchain = false; hists__delete_all_entries(&c2c_ext.function_hists.hists); if (c2c_ext.function_hists.list.fields.next) perf_hpp__reset_output_field(&c2c_ext.function_hists.list);
memset(&c2c_ext.function_hists.stats, 0, sizeof(c2c_ext.function_hists.stats)); c2c_ext.total_cycles = 0; c2c_ext.cl_hists = NULL; c2c_ext.cl_sort = NULL; c2c_ext.symbol_full = false; symbol_conf.use_callchain = saved_use_callchain;}
static bool c2c_function__has_iaddr(const char *cl_sort){ const char *field = cl_sort;
while (field && *field) { const char *end = strchr(field, ','); size_t len = end ? (size_t)(end - field) : strlen(field);
if (len == sizeof("iaddr") - 1 && !strncmp(field, "iaddr", len)) return true; field = end ? end + 1 : NULL; } return false;}
/* * Build the three-level function view in a single pass over the cacheline * entries: * L1: read-side functions (aggregated across all their code addresses) * L2: writing functions contending with each level-1 function * L3: shared cachelines for each function pair */int c2c_function__build(struct c2c_hists *cl_hists, const char *cl_sort, bool symbol_full, struct hists **hists){ static const char output_fields[] = "cycles_percent,total_stores,symbol_view"; static bool hists_initialized; struct rb_node *nd_cl; bool saved_use_callchain; int ret;
if (!hists) return -EINVAL; *hists = NULL;
if (!cl_hists || !cl_sort) return -EINVAL; if (!c2c_function__has_iaddr(cl_sort)) return -EOPNOTSUPP;
saved_use_callchain = symbol_conf.use_callchain; symbol_conf.use_callchain = false; c2c_function__reset();
c2c_ext.cl_hists = cl_hists; c2c_ext.cl_sort = cl_sort; c2c_ext.symbol_full = symbol_full;
/* * __hists__init() (called by c2c_function_hists__init()) also * mutex_init()s the hists lock, so only run it once for this static * hists; on re-entry just re-parse the columns via reinit(). */ if (!hists_initialized) { ret = c2c_function_hists__init(&c2c_ext.function_hists, "symbol_view", 2, NULL); hists_initialized = true; } else { ret = c2c_function_hists__reinit(&c2c_ext.function_hists, /*output=*/NULL, "symbol_view", NULL); } if (ret) goto out_reset;
nd_cl = rb_first_cached(&c2c_ext.cl_hists->hists.entries);
/* An empty C2C report yields an empty (but valid) function view. */ for (; nd_cl; nd_cl = rb_next(nd_cl)) { struct hist_entry *he_cl = rb_entry(nd_cl, struct hist_entry, rb_node); struct c2c_hist_entry *cacheline_he = container_of(he_cl, struct c2c_hist_entry, he); struct thread *synthetic_thread = he_cl->thread;
/* * Detail hists are finalized only for cachelines retained by the * top-level C2C filter. Among those, include any line with sharing * activity, not just HITM. */ if (he_cl->filtered || (c2c_hitm_count(&cacheline_he->stats) == 0 && cacheline_he->stats.tot_peer == 0 && cacheline_he->stats.store == 0 && cacheline_he->stats.load == 0) || !cacheline_he->hists || RB_EMPTY_ROOT(&cacheline_he->hists->hists.entries.rb_root) || !he_cl->mem_info || !synthetic_thread) continue;
ret = c2c_function__process_cl(cacheline_he, synthetic_thread); if (ret) goto out_err; }
ret = c2c_function_hists__reinit(&c2c_ext.function_hists, output_fields, "cycles_percent", NULL); if (ret) goto out_err;
hists__collapse_resort(&c2c_ext.function_hists.hists, NULL); hists__output_resort(&c2c_ext.function_hists.hists, NULL);
c2c_function__finalize();
*hists = &c2c_ext.function_hists.hists; symbol_conf.use_callchain = saved_use_callchain; return 0;
out_err: /* * On error, migrate any entries still in entries_in to entries and * delete them, so a later rebuild does not strand them (the top-level * __hists__init() memset would otherwise lose the pointers). */ hists__collapse_resort(&c2c_ext.function_hists.hists, NULL); hists__output_resort(&c2c_ext.function_hists.hists, NULL);out_reset: c2c_function__reset(); symbol_conf.use_callchain = saved_use_callchain; return ret;}