Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516// SPDX-License-Identifier: GPL-2.0-only/* * intel_pt_decoder.c: Intel Processor Trace support * Copyright (c) 2013-2014, Intel Corporation. */
#ifndef _GNU_SOURCE#define _GNU_SOURCE#endif#include <stdlib.h>#include <stdbool.h>#include <string.h>#include <errno.h>#include <stdint.h>#include <inttypes.h>#include <linux/compiler.h>#include <linux/string.h>#include <linux/zalloc.h>
#include "../auxtrace.h"
#include "intel-pt-insn-decoder.h"#include "intel-pt-pkt-decoder.h"#include "intel-pt-decoder.h"#include "intel-pt-log.h"
#define BITULL(x) (1ULL << (x))
/* IA32_RTIT_CTL MSR bits */#define INTEL_PT_CYC_ENABLE BITULL(1)#define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
#define INTEL_PT_BLK_SIZE 1024
#define BIT63 (((uint64_t)1 << 63))
#define SEVEN_BYTES 0xffffffffffffffULL
#define NO_VMCS 0xffffffffffULL
#define INTEL_PT_RETURN 1
/* * Default maximum number of loops with no packets consumed i.e. stuck in a * loop. */#define INTEL_PT_MAX_LOOPS 100000
struct intel_pt_blk { struct intel_pt_blk *prev; uint64_t ip[INTEL_PT_BLK_SIZE];};
struct intel_pt_stack { struct intel_pt_blk *blk; struct intel_pt_blk *spare; int pos;};
enum intel_pt_p_once { INTEL_PT_PRT_ONCE_UNK_VMCS, INTEL_PT_PRT_ONCE_ERANGE,};
enum intel_pt_pkt_state { INTEL_PT_STATE_NO_PSB, INTEL_PT_STATE_NO_IP, INTEL_PT_STATE_ERR_RESYNC, INTEL_PT_STATE_IN_SYNC, INTEL_PT_STATE_TNT_CONT, INTEL_PT_STATE_TNT, INTEL_PT_STATE_TIP, INTEL_PT_STATE_TIP_PGD, INTEL_PT_STATE_FUP, INTEL_PT_STATE_FUP_NO_TIP, INTEL_PT_STATE_FUP_IN_PSB, INTEL_PT_STATE_RESAMPLE, INTEL_PT_STATE_VM_TIME_CORRELATION,};
static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state){ switch (pkt_state) { case INTEL_PT_STATE_NO_PSB: case INTEL_PT_STATE_NO_IP: case INTEL_PT_STATE_ERR_RESYNC: case INTEL_PT_STATE_IN_SYNC: case INTEL_PT_STATE_TNT_CONT: case INTEL_PT_STATE_RESAMPLE: case INTEL_PT_STATE_VM_TIME_CORRELATION: return true; case INTEL_PT_STATE_TNT: case INTEL_PT_STATE_TIP: case INTEL_PT_STATE_TIP_PGD: case INTEL_PT_STATE_FUP: case INTEL_PT_STATE_FUP_NO_TIP: case INTEL_PT_STATE_FUP_IN_PSB: return false; default: return true; };}
#ifdef INTEL_PT_STRICT#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB#else#define INTEL_PT_STATE_ERR1 (decoder->pkt_state)#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC#endif
struct intel_pt_decoder { int (*get_trace)(struct intel_pt_buffer *buffer, void *data); int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, uint64_t max_insn_cnt, void *data); bool (*pgd_ip)(uint64_t ip, void *data); int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); void *data; struct intel_pt_state state; const unsigned char *buf; size_t len; bool return_compression; bool branch_enable; bool mtc_insn; bool pge; bool have_tma; bool have_cyc; bool fixup_last_mtc; bool have_last_ip; bool in_psb; bool hop; bool leap; bool emulated_ptwrite; bool vm_time_correlation; bool vm_tm_corr_dry_run; bool vm_tm_corr_reliable; bool vm_tm_corr_same_buf; bool vm_tm_corr_continuous; bool nr; bool next_nr; bool iflag; bool next_iflag; enum intel_pt_param_flags flags; uint64_t pos; uint64_t last_ip; uint64_t ip; uint64_t pip_payload; uint64_t timestamp; uint64_t tsc_timestamp; uint64_t ref_timestamp; uint64_t buf_timestamp; uint64_t sample_timestamp; uint64_t ret_addr; uint64_t ctc_timestamp; uint64_t ctc_delta; uint64_t cycle_cnt; uint64_t cyc_ref_timestamp; uint64_t first_timestamp; uint64_t last_reliable_timestamp; uint64_t vmcs; uint64_t print_once; uint64_t last_ctc; uint32_t last_mtc; uint32_t tsc_ctc_ratio_n; uint32_t tsc_ctc_ratio_d; uint32_t tsc_ctc_mult; uint32_t tsc_slip; uint32_t ctc_rem_mask; int mtc_shift; struct intel_pt_stack stack; enum intel_pt_pkt_state pkt_state; enum intel_pt_pkt_ctx pkt_ctx; enum intel_pt_pkt_ctx prev_pkt_ctx; enum intel_pt_blk_type blk_type; int blk_type_pos; struct intel_pt_pkt packet; struct intel_pt_pkt tnt; int pkt_step; int pkt_len; int last_packet_type; unsigned int cbr; unsigned int cbr_seen; unsigned int max_non_turbo_ratio; double max_non_turbo_ratio_fp; double cbr_cyc_to_tsc; double calc_cyc_to_tsc; bool have_calc_cyc_to_tsc; int exec_mode; unsigned int insn_bytes; uint64_t period; enum intel_pt_period_type period_type; uint64_t tot_insn_cnt; uint64_t period_insn_cnt; uint64_t period_mask; uint64_t period_ticks; uint64_t last_masked_timestamp; uint64_t tot_cyc_cnt; uint64_t sample_tot_cyc_cnt; uint64_t base_cyc_cnt; uint64_t cyc_cnt_timestamp; uint64_t ctl; uint64_t cyc_threshold; double tsc_to_cyc; bool continuous_period; bool overflow; bool set_fup_tx_flags; bool set_fup_ptw; bool set_fup_mwait; bool set_fup_pwre; bool set_fup_exstop; bool set_fup_bep; bool set_fup_cfe_ip; bool set_fup_cfe; bool set_fup_mode_exec; bool sample_cyc; unsigned int fup_tx_flags; unsigned int tx_flags; uint64_t fup_ptw_payload; uint64_t fup_mwait_payload; uint64_t fup_pwre_payload; uint64_t cbr_payload; uint64_t timestamp_insn_cnt; uint64_t sample_insn_cnt; uint64_t stuck_ip; struct intel_pt_pkt fup_cfe_pkt; int max_loops; int no_progress; int stuck_ip_prd; int stuck_ip_cnt; uint64_t psb_ip; const unsigned char *next_buf; size_t next_len; unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; int evd_cnt; struct intel_pt_evd evd[INTEL_PT_MAX_EVDS];};
static uint64_t intel_pt_lower_power_of_2(uint64_t x){ int i;
for (i = 0; x != 1; i++) x >>= 1;
return x << i;}
__printf(1, 2)static void p_log(const char *fmt, ...){ char buf[512]; va_list args;
va_start(args, fmt); vsnprintf(buf, sizeof(buf), fmt, args); va_end(args);
fprintf(stderr, "%s\n", buf); intel_pt_log("%s\n", buf);}
static bool intel_pt_print_once(struct intel_pt_decoder *decoder, enum intel_pt_p_once id){ uint64_t bit = 1ULL << id;
if (decoder->print_once & bit) return false; decoder->print_once |= bit; return true;}
static uint64_t intel_pt_cyc_threshold(uint64_t ctl){ if (!(ctl & INTEL_PT_CYC_ENABLE)) return 0;
return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;}
static void intel_pt_setup_period(struct intel_pt_decoder *decoder){ if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { uint64_t period;
period = intel_pt_lower_power_of_2(decoder->period); decoder->period_mask = ~(period - 1); decoder->period_ticks = period; }}
static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d){ if (!d) return 0; return (t / d) * n + ((t % d) * n) / d;}
struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params){ struct intel_pt_decoder *decoder;
if (!params->get_trace || !params->walk_insn) return NULL;
decoder = zalloc(sizeof(struct intel_pt_decoder)); if (!decoder) return NULL;
decoder->get_trace = params->get_trace; decoder->walk_insn = params->walk_insn; decoder->pgd_ip = params->pgd_ip; decoder->lookahead = params->lookahead; decoder->findnew_vmcs_info = params->findnew_vmcs_info; decoder->data = params->data; decoder->return_compression = params->return_compression; decoder->branch_enable = params->branch_enable; decoder->hop = params->quick >= 1; decoder->leap = params->quick >= 2; decoder->vm_time_correlation = params->vm_time_correlation; decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; decoder->first_timestamp = params->first_timestamp; decoder->last_reliable_timestamp = params->first_timestamp; decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
decoder->flags = params->flags;
decoder->ctl = params->ctl; decoder->period = params->period; decoder->period_type = params->period_type;
decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
intel_pt_setup_period(decoder);
decoder->mtc_shift = params->mtc_period; decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
if (!decoder->tsc_ctc_ratio_n) decoder->tsc_ctc_ratio_d = 0;
if (decoder->tsc_ctc_ratio_d) { if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / decoder->tsc_ctc_ratio_d; }
/* * A TSC packet can slip past MTC packets so that the timestamp appears * to go backwards. One estimate is that can be up to about 40 CPU * cycles, which is certainly less than 0x1000 TSC ticks, but accept * slippage an order of magnitude more to be on the safe side. */ decoder->tsc_slip = 0x10000;
intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
if (decoder->hop) intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
return decoder;}
void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, uint64_t first_timestamp){ decoder->first_timestamp = first_timestamp;}
static void intel_pt_pop_blk(struct intel_pt_stack *stack){ struct intel_pt_blk *blk = stack->blk;
stack->blk = blk->prev; if (!stack->spare) stack->spare = blk; else free(blk);}
static uint64_t intel_pt_pop(struct intel_pt_stack *stack){ if (!stack->pos) { if (!stack->blk) return 0; intel_pt_pop_blk(stack); if (!stack->blk) return 0; stack->pos = INTEL_PT_BLK_SIZE; } return stack->blk->ip[--stack->pos];}
static int intel_pt_alloc_blk(struct intel_pt_stack *stack){ struct intel_pt_blk *blk;
if (stack->spare) { blk = stack->spare; stack->spare = NULL; } else { blk = malloc(sizeof(struct intel_pt_blk)); if (!blk) return -ENOMEM; }
blk->prev = stack->blk; stack->blk = blk; stack->pos = 0; return 0;}
static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip){ int err;
if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { err = intel_pt_alloc_blk(stack); if (err) return err; }
stack->blk->ip[stack->pos++] = ip; return 0;}
static void intel_pt_clear_stack(struct intel_pt_stack *stack){ while (stack->blk) intel_pt_pop_blk(stack); stack->pos = 0;}
static void intel_pt_free_stack(struct intel_pt_stack *stack){ intel_pt_clear_stack(stack); zfree(&stack->blk); zfree(&stack->spare);}
void intel_pt_decoder_free(struct intel_pt_decoder *decoder){ intel_pt_free_stack(&decoder->stack); free(decoder);}
static int intel_pt_ext_err(int code){ switch (code) { case -ENOMEM: return INTEL_PT_ERR_NOMEM; case -ENOSYS: return INTEL_PT_ERR_INTERN; case -EBADMSG: return INTEL_PT_ERR_BADPKT; case -ENODATA: return INTEL_PT_ERR_NODATA; case -EILSEQ: return INTEL_PT_ERR_NOINSN; case -ENOENT: return INTEL_PT_ERR_MISMAT; case -EOVERFLOW: return INTEL_PT_ERR_OVR; case -ENOSPC: return INTEL_PT_ERR_LOST; case -ELOOP: return INTEL_PT_ERR_NELOOP; case -ECONNRESET: return INTEL_PT_ERR_EPTW; default: return INTEL_PT_ERR_UNK; }}
static const char *intel_pt_err_msgs[] = { [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", [INTEL_PT_ERR_INTERN] = "Internal error", [INTEL_PT_ERR_BADPKT] = "Bad packet", [INTEL_PT_ERR_NODATA] = "No more data", [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", [INTEL_PT_ERR_OVR] = "Overflow packet", [INTEL_PT_ERR_LOST] = "Lost trace data", [INTEL_PT_ERR_UNK] = "Unknown error!", [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", [INTEL_PT_ERR_EPTW] = "Broken emulated ptwrite",};
int intel_pt__strerror(int code, char *buf, size_t buflen){ if (code < 1 || code >= INTEL_PT_ERR_MAX) code = INTEL_PT_ERR_UNK; strlcpy(buf, intel_pt_err_msgs[code], buflen); return 0;}
static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, uint64_t last_ip){ uint64_t ip;
switch (packet->count) { case 1: ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | packet->payload; break; case 2: ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | packet->payload; break; case 3: ip = packet->payload; /* Sign-extend 6-byte ip */ if (ip & (uint64_t)0x800000000000ULL) ip |= (uint64_t)0xffff000000000000ULL; break; case 4: ip = (last_ip & (uint64_t)0xffff000000000000ULL) | packet->payload; break; case 6: ip = packet->payload; break; default: return 0; }
return ip;}
static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder){ decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); decoder->have_last_ip = true;}
static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder){ intel_pt_set_last_ip(decoder); decoder->ip = decoder->last_ip;}
static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder){ intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, decoder->buf);}
static int intel_pt_bug(struct intel_pt_decoder *decoder){ intel_pt_log("ERROR: Internal error\n"); decoder->pkt_state = INTEL_PT_STATE_NO_PSB; return -ENOSYS;}
static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder){ decoder->tx_flags = 0;}
static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder){ decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;}
static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder){ decoder->pip_payload = decoder->packet.payload;}
static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder){ decoder->next_nr = decoder->pip_payload & 1;}
static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder){ decoder->nr = decoder->pip_payload & 1; decoder->next_nr = decoder->nr;}
static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder){ intel_pt_update_pip(decoder); intel_pt_set_nr(decoder);}
static int intel_pt_bad_packet(struct intel_pt_decoder *decoder){ intel_pt_clear_tx_flags(decoder); decoder->have_tma = false; decoder->pkt_len = 1; decoder->pkt_step = 1; intel_pt_decoder_log_packet(decoder); if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { intel_pt_log("ERROR: Bad packet\n"); decoder->pkt_state = INTEL_PT_STATE_ERR1; } return -EBADMSG;}
static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder){ decoder->sample_timestamp = decoder->timestamp; decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; decoder->state.cycles = decoder->tot_cyc_cnt;}
static void intel_pt_reposition(struct intel_pt_decoder *decoder){ decoder->ip = 0; decoder->pkt_state = INTEL_PT_STATE_NO_PSB; decoder->timestamp = 0; decoder->have_tma = false;}
static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition){ struct intel_pt_buffer buffer = { .buf = 0, }; int ret;
decoder->pkt_step = 0;
intel_pt_log("Getting more data\n"); ret = decoder->get_trace(&buffer, decoder->data); if (ret) return ret; decoder->buf = buffer.buf; decoder->len = buffer.len; if (!decoder->len) { intel_pt_log("No more data\n"); return -ENODATA; } decoder->buf_timestamp = buffer.ref_timestamp; if (!buffer.consecutive || reposition) { intel_pt_reposition(decoder); decoder->ref_timestamp = buffer.ref_timestamp; decoder->state.trace_nr = buffer.trace_nr; decoder->vm_tm_corr_same_buf = false; intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", decoder->ref_timestamp); return -ENOLINK; }
return 0;}
static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, bool reposition){ if (!decoder->next_buf) return intel_pt_get_data(decoder, reposition);
decoder->buf = decoder->next_buf; decoder->len = decoder->next_len; decoder->next_buf = 0; decoder->next_len = 0; return 0;}
static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder){ unsigned char *buf = decoder->temp_buf; size_t old_len, len, n; int ret;
old_len = decoder->len; len = decoder->len; memcpy(buf, decoder->buf, len);
ret = intel_pt_get_data(decoder, false); if (ret) { decoder->pos += old_len; return ret < 0 ? ret : -EINVAL; }
n = INTEL_PT_PKT_MAX_SZ - len; if (n > decoder->len) n = decoder->len; memcpy(buf + len, decoder->buf, n); len += n;
decoder->prev_pkt_ctx = decoder->pkt_ctx; ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); if (ret < (int)old_len) { decoder->next_buf = decoder->buf; decoder->next_len = decoder->len; decoder->buf = buf; decoder->len = old_len; return intel_pt_bad_packet(decoder); }
decoder->next_buf = decoder->buf + (ret - old_len); decoder->next_len = decoder->len - (ret - old_len);
decoder->buf = buf; decoder->len = ret;
return ret;}
struct intel_pt_pkt_info { struct intel_pt_decoder *decoder; struct intel_pt_pkt packet; uint64_t pos; int pkt_len; int last_packet_type; void *data;};
typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
/* Lookahead packets in current buffer */static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, intel_pt_pkt_cb_t cb, void *data){ struct intel_pt_pkt_info pkt_info; const unsigned char *buf = decoder->buf; enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; size_t len = decoder->len; int ret;
pkt_info.decoder = decoder; pkt_info.pos = decoder->pos; pkt_info.pkt_len = decoder->pkt_step; pkt_info.last_packet_type = decoder->last_packet_type; pkt_info.data = data;
while (1) { do { pkt_info.pos += pkt_info.pkt_len; buf += pkt_info.pkt_len; len -= pkt_info.pkt_len;
if (!len) return INTEL_PT_NEED_MORE_BYTES;
ret = intel_pt_get_packet(buf, len, &pkt_info.packet, &pkt_ctx); if (!ret) return INTEL_PT_NEED_MORE_BYTES; if (ret < 0) return ret;
pkt_info.pkt_len = ret; } while (pkt_info.packet.type == INTEL_PT_PAD);
ret = cb(&pkt_info); if (ret) return 0;
pkt_info.last_packet_type = pkt_info.packet.type; }}
struct intel_pt_calc_cyc_to_tsc_info { uint64_t cycle_cnt; unsigned int cbr; uint32_t last_mtc; uint64_t ctc_timestamp; uint64_t ctc_delta; uint64_t tsc_timestamp; uint64_t timestamp; bool have_tma; bool fixup_last_mtc; bool from_mtc; double cbr_cyc_to_tsc;};
/* * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA * packet by copying the missing bits from the current MTC assuming the least * difference between the two, and that the current MTC comes after last_mtc. */static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, uint32_t *last_mtc){ uint32_t first_missing_bit = 1U << (16 - mtc_shift); uint32_t mask = ~(first_missing_bit - 1);
*last_mtc |= mtc & mask; if (*last_mtc >= mtc) { *last_mtc -= first_missing_bit; *last_mtc &= 0xff; }}
static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info){ struct intel_pt_decoder *decoder = pkt_info->decoder; struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; uint64_t timestamp; double cyc_to_tsc; unsigned int cbr; uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
switch (pkt_info->packet.type) { case INTEL_PT_TNT: case INTEL_PT_TIP_PGE: case INTEL_PT_TIP: case INTEL_PT_FUP: case INTEL_PT_PSB: case INTEL_PT_PIP: case INTEL_PT_MODE_EXEC: case INTEL_PT_MODE_TSX: case INTEL_PT_PSBEND: case INTEL_PT_PAD: case INTEL_PT_VMCS: case INTEL_PT_MNT: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: return 0;
case INTEL_PT_MTC: if (!data->have_tma) return 0;
mtc = pkt_info->packet.payload; if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { data->fixup_last_mtc = false; intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, &data->last_mtc); } if (mtc > data->last_mtc) mtc_delta = mtc - data->last_mtc; else mtc_delta = mtc + 256 - data->last_mtc; data->ctc_delta += mtc_delta << decoder->mtc_shift; data->last_mtc = mtc;
if (decoder->tsc_ctc_mult) { timestamp = data->ctc_timestamp + data->ctc_delta * decoder->tsc_ctc_mult; } else { timestamp = data->ctc_timestamp + multdiv(data->ctc_delta, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); }
if (timestamp < data->timestamp) return 1;
if (pkt_info->last_packet_type != INTEL_PT_CYC) { data->timestamp = timestamp; return 0; }
break;
case INTEL_PT_TSC: /* * For now, do not support using TSC packets - refer * intel_pt_calc_cyc_to_tsc(). */ if (data->from_mtc) return 1; timestamp = pkt_info->packet.payload | (data->timestamp & (0xffULL << 56)); if (data->from_mtc && timestamp < data->timestamp && data->timestamp - timestamp < decoder->tsc_slip) return 1; if (timestamp < data->timestamp) timestamp += (1ULL << 56); if (pkt_info->last_packet_type != INTEL_PT_CYC) { if (data->from_mtc) return 1; data->tsc_timestamp = timestamp; data->timestamp = timestamp; return 0; } break;
case INTEL_PT_TMA: if (data->from_mtc) return 1;
if (!decoder->tsc_ctc_ratio_d) return 0;
ctc = pkt_info->packet.payload; fc = pkt_info->packet.count; ctc_rem = ctc & decoder->ctc_rem_mask;
data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
data->ctc_timestamp = data->tsc_timestamp - fc; if (decoder->tsc_ctc_mult) { data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; } else { data->ctc_timestamp -= multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); }
data->ctc_delta = 0; data->have_tma = true; data->fixup_last_mtc = true;
return 0;
case INTEL_PT_CYC: data->cycle_cnt += pkt_info->packet.payload; return 0;
case INTEL_PT_CBR: cbr = pkt_info->packet.payload; if (data->cbr && data->cbr != cbr) return 1; data->cbr = cbr; data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; return 0;
case INTEL_PT_TIP_PGD: case INTEL_PT_TRACESTOP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_OVF: case INTEL_PT_BAD: /* Does not happen */ default: return 1; }
if (!data->cbr && decoder->cbr) { data->cbr = decoder->cbr; data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; }
if (!data->cycle_cnt) return 1;
cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); return 1; }
decoder->calc_cyc_to_tsc = cyc_to_tsc; decoder->have_calc_cyc_to_tsc = true;
if (data->cbr) { intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); } else { intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", cyc_to_tsc, pkt_info->pos); }
return 1;}
static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, bool from_mtc){ struct intel_pt_calc_cyc_to_tsc_info data = { .cycle_cnt = 0, .cbr = 0, .last_mtc = decoder->last_mtc, .ctc_timestamp = decoder->ctc_timestamp, .ctc_delta = decoder->ctc_delta, .tsc_timestamp = decoder->tsc_timestamp, .timestamp = decoder->timestamp, .have_tma = decoder->have_tma, .fixup_last_mtc = decoder->fixup_last_mtc, .from_mtc = from_mtc, .cbr_cyc_to_tsc = 0, };
/* * For now, do not support using TSC packets for at least the reasons: * 1) timing might have stopped * 2) TSC packets within PSB+ can slip against CYC packets */ if (!from_mtc) return;
intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);}
static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder){ int ret;
decoder->last_packet_type = decoder->packet.type;
do { decoder->pos += decoder->pkt_step; decoder->buf += decoder->pkt_step; decoder->len -= decoder->pkt_step;
if (!decoder->len) { ret = intel_pt_get_next_data(decoder, false); if (ret) return ret; }
decoder->prev_pkt_ctx = decoder->pkt_ctx; ret = intel_pt_get_packet(decoder->buf, decoder->len, &decoder->packet, &decoder->pkt_ctx); if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { ret = intel_pt_get_split_packet(decoder); if (ret < 0) return ret; } if (ret <= 0) return intel_pt_bad_packet(decoder);
decoder->pkt_len = ret; decoder->pkt_step = ret; intel_pt_decoder_log_packet(decoder); } while (decoder->packet.type == INTEL_PT_PAD);
return 0;}
static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder){ uint64_t timestamp, masked_timestamp;
timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; masked_timestamp = timestamp & decoder->period_mask; if (decoder->continuous_period) { if (masked_timestamp > decoder->last_masked_timestamp) return 1; } else { timestamp += 1; masked_timestamp = timestamp & decoder->period_mask; if (masked_timestamp > decoder->last_masked_timestamp) { decoder->last_masked_timestamp = masked_timestamp; decoder->continuous_period = true; } }
if (masked_timestamp < decoder->last_masked_timestamp) return decoder->period_ticks;
return decoder->period_ticks - (timestamp - masked_timestamp);}
static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder){ switch (decoder->period_type) { case INTEL_PT_PERIOD_INSTRUCTIONS: return decoder->period - decoder->period_insn_cnt; case INTEL_PT_PERIOD_TICKS: return intel_pt_next_period(decoder); case INTEL_PT_PERIOD_NONE: case INTEL_PT_PERIOD_MTC: default: return 0; }}
static void intel_pt_sample_insn(struct intel_pt_decoder *decoder){ uint64_t timestamp, masked_timestamp;
switch (decoder->period_type) { case INTEL_PT_PERIOD_INSTRUCTIONS: decoder->period_insn_cnt = 0; break; case INTEL_PT_PERIOD_TICKS: timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; masked_timestamp = timestamp & decoder->period_mask; if (masked_timestamp > decoder->last_masked_timestamp) decoder->last_masked_timestamp = masked_timestamp; else decoder->last_masked_timestamp += decoder->period_ticks; break; case INTEL_PT_PERIOD_NONE: case INTEL_PT_PERIOD_MTC: default: break; }
decoder->state.type |= INTEL_PT_INSTRUCTION;}
/* * Sample FUP instruction at the same time as reporting the FUP event, so the * instruction sample gets the same flags as the FUP event. */static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder){ struct intel_pt_insn intel_pt_insn; uint64_t max_insn_cnt, insn_cnt = 0; int err;
decoder->state.insn_op = INTEL_PT_OP_OTHER; decoder->state.insn_len = 0;
if (!decoder->branch_enable || !decoder->pge || decoder->hop || decoder->ip != decoder->last_ip) return;
if (!decoder->mtc_insn) decoder->mtc_insn = true;
max_insn_cnt = intel_pt_next_sample(decoder); if (max_insn_cnt != 1) return;
err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip, 0, max_insn_cnt, decoder->data); /* Ignore error, it will be reported next walk anyway */ if (err) return;
if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) { intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip); return; }
decoder->tot_insn_cnt += insn_cnt; decoder->timestamp_insn_cnt += insn_cnt; decoder->sample_insn_cnt += insn_cnt; decoder->period_insn_cnt += insn_cnt;
intel_pt_sample_insn(decoder);
decoder->state.type |= INTEL_PT_INSTRUCTION; decoder->ip += intel_pt_insn.length;}
static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, struct intel_pt_insn *intel_pt_insn, uint64_t ip){ uint64_t max_insn_cnt, insn_cnt = 0; int err;
if (!decoder->mtc_insn) decoder->mtc_insn = true;
max_insn_cnt = intel_pt_next_sample(decoder);
err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, max_insn_cnt, decoder->data);
decoder->tot_insn_cnt += insn_cnt; decoder->timestamp_insn_cnt += insn_cnt; decoder->sample_insn_cnt += insn_cnt; decoder->period_insn_cnt += insn_cnt;
if (err) { decoder->no_progress = 0; decoder->pkt_state = INTEL_PT_STATE_ERR2; intel_pt_log_at("ERROR: Failed to get instruction", decoder->ip); if (err == -ENOENT) return -ENOLINK; return -EILSEQ; }
if (ip && decoder->ip == ip) { err = -EAGAIN; goto out; }
if (max_insn_cnt && insn_cnt >= max_insn_cnt) intel_pt_sample_insn(decoder);
if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { decoder->state.type = INTEL_PT_INSTRUCTION; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->ip += intel_pt_insn->length; err = INTEL_PT_RETURN; goto out; }
if (intel_pt_insn->op == INTEL_PT_OP_CALL) { /* Zero-length calls are excluded */ if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || intel_pt_insn->rel) { err = intel_pt_push(&decoder->stack, decoder->ip + intel_pt_insn->length); if (err) goto out; } } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { decoder->ret_addr = intel_pt_pop(&decoder->stack); }
if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { int cnt = decoder->no_progress++;
decoder->state.from_ip = decoder->ip; decoder->ip += intel_pt_insn->length + intel_pt_insn->rel; decoder->state.to_ip = decoder->ip; err = INTEL_PT_RETURN;
/* * Check for being stuck in a loop. This can happen if a * decoder error results in the decoder erroneously setting the * ip to an address that is itself in an infinite loop that * consumes no packets. When that happens, there must be an * unconditional branch. */ if (cnt) { if (cnt == 1) { decoder->stuck_ip = decoder->state.to_ip; decoder->stuck_ip_prd = 1; decoder->stuck_ip_cnt = 1; } else if (cnt > decoder->max_loops || decoder->state.to_ip == decoder->stuck_ip) { intel_pt_log_at("ERROR: Never-ending loop", decoder->state.to_ip); decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; err = -ELOOP; goto out; } else if (!--decoder->stuck_ip_cnt) { decoder->stuck_ip_prd += 1; decoder->stuck_ip_cnt = decoder->stuck_ip_prd; decoder->stuck_ip = decoder->state.to_ip; } } goto out_no_progress; }out: decoder->no_progress = 0;out_no_progress: decoder->state.insn_op = intel_pt_insn->op; decoder->state.insn_len = intel_pt_insn->length; memcpy(decoder->state.insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
if (decoder->tx_flags & INTEL_PT_IN_TX) decoder->state.flags |= INTEL_PT_IN_TX;
return err;}
static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder){ bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
decoder->exec_mode = decoder->packet.payload; decoder->iflag = iflag; decoder->next_iflag = iflag; decoder->state.from_iflag = iflag; decoder->state.to_iflag = iflag;}
static void intel_pt_mode_exec(struct intel_pt_decoder *decoder){ bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
decoder->exec_mode = decoder->packet.payload; decoder->next_iflag = iflag;}
static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder){ decoder->state.type |= INTEL_PT_IFLAG_CHG; decoder->state.from_iflag = decoder->iflag; decoder->state.to_iflag = decoder->next_iflag; decoder->iflag = decoder->next_iflag;}
static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder){ if (decoder->iflag != decoder->next_iflag) intel_pt_sample_iflag(decoder);}
static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder){ decoder->set_fup_tx_flags = false; decoder->set_fup_ptw = false; decoder->set_fup_mwait = false; decoder->set_fup_pwre = false; decoder->set_fup_exstop = false; decoder->set_fup_bep = false; decoder->set_fup_cfe_ip = false; decoder->set_fup_cfe = false; decoder->evd_cnt = 0; decoder->set_fup_mode_exec = false; decoder->iflag = decoder->next_iflag;}
static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip){ enum intel_pt_sample_type type = decoder->state.type; bool sample_fup_insn = false; bool ret = false;
decoder->state.type &= ~INTEL_PT_BRANCH; decoder->state.insn_op = INTEL_PT_OP_OTHER; decoder->state.insn_len = 0;
if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) { bool ip = decoder->set_fup_cfe_ip;
decoder->set_fup_cfe_ip = false; decoder->set_fup_cfe = false; decoder->state.type |= INTEL_PT_EVT; if (!ip && decoder->pge) decoder->state.type |= INTEL_PT_BRANCH; decoder->state.cfe_type = decoder->fup_cfe_pkt.count; decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload; decoder->state.evd_cnt = decoder->evd_cnt; decoder->state.evd = decoder->evd; decoder->evd_cnt = 0; if (ip || decoder->pge) decoder->state.flags |= INTEL_PT_FUP_IP; ret = true; } if (decoder->set_fup_mode_exec) { decoder->set_fup_mode_exec = false; intel_pt_sample_iflag(decoder); sample_fup_insn = no_tip; ret = true; } if (decoder->set_fup_tx_flags) { decoder->set_fup_tx_flags = false; decoder->tx_flags = decoder->fup_tx_flags; decoder->state.type |= INTEL_PT_TRANSACTION; if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) decoder->state.type |= INTEL_PT_BRANCH; decoder->state.flags = decoder->fup_tx_flags; ret = true; } if (decoder->set_fup_ptw) { decoder->set_fup_ptw = false; decoder->state.type |= INTEL_PT_PTW; decoder->state.flags |= INTEL_PT_FUP_IP; decoder->state.ptw_payload = decoder->fup_ptw_payload; ret = true; } if (decoder->set_fup_mwait) { decoder->set_fup_mwait = false; decoder->state.type |= INTEL_PT_MWAIT_OP; decoder->state.mwait_payload = decoder->fup_mwait_payload; ret = true; } if (decoder->set_fup_pwre) { decoder->set_fup_pwre = false; decoder->state.type |= INTEL_PT_PWR_ENTRY; decoder->state.pwre_payload = decoder->fup_pwre_payload; ret = true; } if (decoder->set_fup_exstop) { decoder->set_fup_exstop = false; decoder->state.type |= INTEL_PT_EX_STOP; decoder->state.flags |= INTEL_PT_FUP_IP; ret = true; } if (decoder->set_fup_bep) { decoder->set_fup_bep = false; decoder->state.type |= INTEL_PT_BLK_ITEMS; ret = true; } if (decoder->overflow) { decoder->overflow = false; if (!ret && !decoder->pge) { if (decoder->hop) { decoder->state.type = 0; decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; } decoder->pge = true; decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; decoder->state.from_ip = 0; decoder->state.to_ip = decoder->ip; return true; } } if (ret) { decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; if (sample_fup_insn) intel_pt_sample_fup_insn(decoder); } else { decoder->state.type = type; } return ret;}
static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, struct intel_pt_insn *intel_pt_insn, uint64_t ip, int err){ return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && ip == decoder->ip + intel_pt_insn->length;}
static int intel_pt_walk_fup(struct intel_pt_decoder *decoder){ struct intel_pt_insn intel_pt_insn; uint64_t ip; int err;
ip = decoder->last_ip;
while (1) { err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); if (err == INTEL_PT_RETURN) return 0; if (err == -EAGAIN || intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; if (intel_pt_fup_event(decoder, no_tip) && no_tip) return 0; return -EAGAIN; } decoder->set_fup_tx_flags = false; if (err) return err;
if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { intel_pt_log_at("ERROR: Unexpected indirect branch", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; return -ENOENT; }
if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { intel_pt_log_at("ERROR: Unexpected conditional branch", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; return -ENOENT; }
intel_pt_bug(decoder); }}
static int intel_pt_walk_tip(struct intel_pt_decoder *decoder){ struct intel_pt_insn intel_pt_insn; int err;
err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); if (err == INTEL_PT_RETURN && decoder->pgd_ip && decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && (decoder->state.type & INTEL_PT_BRANCH) && decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { /* Unconditional branch leaving filter region */ decoder->no_progress = 0; decoder->pge = false; decoder->continuous_period = false; decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->state.type |= INTEL_PT_TRACE_END; intel_pt_update_nr(decoder); return 0; } if (err == INTEL_PT_RETURN) return 0; if (err) return err;
intel_pt_update_nr(decoder); intel_pt_sample_iflag_chg(decoder);
if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { decoder->pge = false; decoder->continuous_period = false; decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->state.from_ip = decoder->ip; if (decoder->packet.count == 0) { decoder->state.to_ip = 0; } else { decoder->state.to_ip = decoder->last_ip; decoder->ip = decoder->last_ip; } decoder->state.type |= INTEL_PT_TRACE_END; } else { decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->state.from_ip = decoder->ip; if (decoder->packet.count == 0) { decoder->state.to_ip = 0; } else { decoder->state.to_ip = decoder->last_ip; decoder->ip = decoder->last_ip; } } return 0; }
if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { uint64_t to_ip = decoder->ip + intel_pt_insn.length + intel_pt_insn.rel;
if (decoder->pgd_ip && decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && decoder->pgd_ip(to_ip, decoder->data)) { /* Conditional branch leaving filter region */ decoder->pge = false; decoder->continuous_period = false; decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->ip = to_ip; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = to_ip; decoder->state.type |= INTEL_PT_TRACE_END; return 0; } intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; return -ENOENT; }
return intel_pt_bug(decoder);}
struct eptw_data { int bit_countdown; uint64_t payload;};
static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info){ struct eptw_data *data = pkt_info->data; int nr_bits;
switch (pkt_info->packet.type) { case INTEL_PT_PAD: case INTEL_PT_MNT: case INTEL_PT_MODE_EXEC: case INTEL_PT_MODE_TSX: case INTEL_PT_MTC: case INTEL_PT_FUP: case INTEL_PT_CYC: case INTEL_PT_CBR: case INTEL_PT_TSC: case INTEL_PT_TMA: case INTEL_PT_PIP: case INTEL_PT_VMCS: case INTEL_PT_PSB: case INTEL_PT_PSBEND: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: break;
case INTEL_PT_TNT: nr_bits = data->bit_countdown; if (nr_bits > pkt_info->packet.count) nr_bits = pkt_info->packet.count; data->payload <<= nr_bits; data->payload |= pkt_info->packet.payload >> (64 - nr_bits); data->bit_countdown -= nr_bits; return !data->bit_countdown;
case INTEL_PT_TIP_PGE: case INTEL_PT_TIP_PGD: case INTEL_PT_TIP: case INTEL_PT_BAD: case INTEL_PT_OVF: case INTEL_PT_TRACESTOP: default: return 1; }
return 0;}
static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder){ int n = 64 - decoder->tnt.count; struct eptw_data data = { .bit_countdown = n, .payload = decoder->tnt.payload >> n, };
decoder->emulated_ptwrite = false; intel_pt_log("Emulated ptwrite detected\n");
intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data); if (data.bit_countdown) return -ECONNRESET;
decoder->state.type = INTEL_PT_PTW; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->state.ptw_payload = data.payload; return 0;}
static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder){ struct intel_pt_insn intel_pt_insn; int err;
while (1) { if (decoder->emulated_ptwrite) return intel_pt_emulated_ptwrite(decoder); err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); if (err == INTEL_PT_RETURN) { decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite; return 0; } if (err) { decoder->emulated_ptwrite = false; return err; }
if (intel_pt_insn.op == INTEL_PT_OP_RET) { if (!decoder->return_compression) { intel_pt_log_at("ERROR: RET when expecting conditional branch", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR3; return -ENOENT; } if (!decoder->ret_addr) { intel_pt_log_at("ERROR: Bad RET compression (stack empty)", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR3; return -ENOENT; } if (!(decoder->tnt.payload & BIT63)) { intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR3; return -ENOENT; } decoder->tnt.count -= 1; if (decoder->tnt.count) decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; else decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->tnt.payload <<= 1; decoder->state.from_ip = decoder->ip; decoder->ip = decoder->ret_addr; decoder->state.to_ip = decoder->ip; return 0; }
if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { /* Handle deferred TIPs */ err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type != INTEL_PT_TIP || decoder->packet.count == 0) { intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", decoder->ip); decoder->pkt_state = INTEL_PT_STATE_ERR3; decoder->pkt_step = 0; return -ENOENT; } intel_pt_set_last_ip(decoder); decoder->state.from_ip = decoder->ip; decoder->state.to_ip = decoder->last_ip; decoder->ip = decoder->last_ip; intel_pt_update_nr(decoder); intel_pt_sample_iflag_chg(decoder); return 0; }
if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { decoder->tnt.count -= 1; if (decoder->tnt.count) decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; else decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; if (decoder->tnt.payload & BIT63) { decoder->tnt.payload <<= 1; decoder->state.from_ip = decoder->ip; decoder->ip += intel_pt_insn.length + intel_pt_insn.rel; decoder->state.to_ip = decoder->ip; return 0; } /* Instruction sample for a non-taken branch */ if (decoder->state.type & INTEL_PT_INSTRUCTION) { decoder->tnt.payload <<= 1; decoder->state.type = INTEL_PT_INSTRUCTION; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->ip += intel_pt_insn.length; return 0; } decoder->sample_cyc = false; decoder->ip += intel_pt_insn.length; if (!decoder->tnt.count) { intel_pt_update_sample_time(decoder); return -EAGAIN; } decoder->tnt.payload <<= 1; continue; }
return intel_pt_bug(decoder); }}
static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip){ unsigned int fup_tx_flags; int err;
fup_tx_flags = decoder->packet.payload & (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type == INTEL_PT_FUP) { decoder->fup_tx_flags = fup_tx_flags; decoder->set_fup_tx_flags = true; if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) *no_tip = true; } else { intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", decoder->pos); intel_pt_update_in_tx(decoder); } return 0;}
static int intel_pt_evd(struct intel_pt_decoder *decoder){ if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) { intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos); return -ENOSYS; } decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){ .type = decoder->packet.count, .payload = decoder->packet.payload, }; return 0;}
static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp){ timestamp |= (ref_timestamp & (0xffULL << 56));
if (timestamp < ref_timestamp) { if (ref_timestamp - timestamp > (1ULL << 55)) timestamp += (1ULL << 56); } else { if (timestamp - ref_timestamp > (1ULL << 55)) timestamp -= (1ULL << 56); }
return timestamp;}
/* For use only when decoder->vm_time_correlation is true */static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, uint64_t timestamp){ uint64_t max_timestamp = decoder->buf_timestamp;
if (!max_timestamp) { max_timestamp = decoder->last_reliable_timestamp + 0x400000000ULL; } return timestamp >= decoder->last_reliable_timestamp && timestamp < decoder->buf_timestamp;}
static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder){ uint64_t timestamp; bool bad = false;
decoder->have_tma = false;
if (decoder->ref_timestamp) { timestamp = intel_pt_8b_tsc(decoder->packet.payload, decoder->ref_timestamp); decoder->tsc_timestamp = timestamp; decoder->timestamp = timestamp; decoder->ref_timestamp = 0; decoder->timestamp_insn_cnt = 0; } else if (decoder->timestamp) { timestamp = decoder->packet.payload | (decoder->timestamp & (0xffULL << 56)); decoder->tsc_timestamp = timestamp; if (timestamp < decoder->timestamp && decoder->timestamp - timestamp < decoder->tsc_slip) { intel_pt_log_to("Suppressing backwards timestamp", timestamp); timestamp = decoder->timestamp; } if (timestamp < decoder->timestamp) { if (!decoder->buf_timestamp || (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { intel_pt_log_to("Wraparound timestamp", timestamp); timestamp += (1ULL << 56); decoder->tsc_timestamp = timestamp; } else { intel_pt_log_to("Suppressing bad timestamp", timestamp); timestamp = decoder->timestamp; bad = true; } } if (decoder->vm_time_correlation && (bad || !intel_pt_time_in_range(decoder, timestamp)) && intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) p_log("Timestamp out of range"); decoder->timestamp = timestamp; decoder->timestamp_insn_cnt = 0; }
if (decoder->last_packet_type == INTEL_PT_CYC) { decoder->cyc_ref_timestamp = decoder->timestamp; decoder->cycle_cnt = 0; decoder->have_calc_cyc_to_tsc = false; intel_pt_calc_cyc_to_tsc(decoder, false); }
intel_pt_log_to("Setting timestamp", decoder->timestamp);}
static int intel_pt_overflow(struct intel_pt_decoder *decoder){ intel_pt_log("ERROR: Buffer overflow\n"); intel_pt_clear_tx_flags(decoder); intel_pt_set_nr(decoder); decoder->timestamp_insn_cnt = 0; decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->state.from_ip = decoder->ip; decoder->ip = 0; decoder->pge = false; intel_pt_clear_fup_event(decoder); decoder->overflow = true; return -EOVERFLOW;}
static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder){ if (decoder->have_cyc) return;
decoder->cyc_cnt_timestamp = decoder->timestamp; decoder->base_cyc_cnt = decoder->tot_cyc_cnt;}
static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder){ decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
if (decoder->pge) intel_pt_mtc_cyc_cnt_pge(decoder);}
static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder){ uint64_t tot_cyc_cnt, tsc_delta;
if (decoder->have_cyc) return;
decoder->sample_cyc = true;
if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) return;
tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
if (tot_cyc_cnt > decoder->tot_cyc_cnt) decoder->tot_cyc_cnt = tot_cyc_cnt;}
static void intel_pt_calc_tma(struct intel_pt_decoder *decoder){ uint32_t ctc = decoder->packet.payload; uint32_t fc = decoder->packet.count; uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
if (!decoder->tsc_ctc_ratio_d) return;
if (decoder->pge && !decoder->in_psb) intel_pt_mtc_cyc_cnt_pge(decoder); else intel_pt_mtc_cyc_cnt_upd(decoder);
decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; decoder->last_ctc = ctc - ctc_rem; decoder->ctc_timestamp = decoder->tsc_timestamp - fc; if (decoder->tsc_ctc_mult) { decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; } else { decoder->ctc_timestamp -= multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); } decoder->ctc_delta = 0; decoder->have_tma = true; decoder->fixup_last_mtc = true; intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);}
static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder){ uint64_t timestamp; uint32_t mtc, mtc_delta;
if (!decoder->have_tma) return;
mtc = decoder->packet.payload;
if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { decoder->fixup_last_mtc = false; intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, &decoder->last_mtc); }
if (mtc > decoder->last_mtc) mtc_delta = mtc - decoder->last_mtc; else mtc_delta = mtc + 256 - decoder->last_mtc;
decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
if (decoder->tsc_ctc_mult) { timestamp = decoder->ctc_timestamp + decoder->ctc_delta * decoder->tsc_ctc_mult; } else { timestamp = decoder->ctc_timestamp + multdiv(decoder->ctc_delta, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); }
if (timestamp < decoder->timestamp) intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", timestamp, decoder->timestamp); else decoder->timestamp = timestamp;
intel_pt_mtc_cyc_cnt_upd(decoder);
decoder->timestamp_insn_cnt = 0; decoder->last_mtc = mtc;
if (decoder->last_packet_type == INTEL_PT_CYC) { decoder->cyc_ref_timestamp = decoder->timestamp; decoder->cycle_cnt = 0; decoder->have_calc_cyc_to_tsc = false; intel_pt_calc_cyc_to_tsc(decoder, true); }
intel_pt_log_to("Setting timestamp", decoder->timestamp);}
static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder){ unsigned int cbr = decoder->packet.payload & 0xff;
decoder->cbr_payload = decoder->packet.payload;
if (decoder->cbr == cbr) return;
decoder->cbr = cbr; decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; decoder->cyc_ref_timestamp = decoder->timestamp; decoder->cycle_cnt = 0;
intel_pt_mtc_cyc_cnt_cbr(decoder);}
static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder){ uint64_t timestamp = decoder->cyc_ref_timestamp;
decoder->have_cyc = true;
decoder->cycle_cnt += decoder->packet.payload; if (decoder->pge) decoder->tot_cyc_cnt += decoder->packet.payload; decoder->sample_cyc = true;
if (!decoder->cyc_ref_timestamp) return;
if (decoder->have_calc_cyc_to_tsc) timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; else if (decoder->cbr) timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; else return;
if (timestamp < decoder->timestamp) intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", timestamp, decoder->timestamp); else decoder->timestamp = timestamp;
decoder->timestamp_insn_cnt = 0;
intel_pt_log_to("Setting timestamp", decoder->timestamp);}
static void intel_pt_bbp(struct intel_pt_decoder *decoder){ if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); decoder->state.items.is_32_bit = false; } decoder->blk_type = decoder->packet.payload; decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); if (decoder->blk_type == INTEL_PT_GP_REGS) decoder->state.items.is_32_bit = decoder->packet.count; if (decoder->blk_type_pos < 0) { intel_pt_log("WARNING: Unknown block type %u\n", decoder->blk_type); } else if (decoder->state.items.mask[decoder->blk_type_pos]) { intel_pt_log("WARNING: Duplicate block type %u\n", decoder->blk_type); }}
static void intel_pt_bip(struct intel_pt_decoder *decoder){ uint32_t id = decoder->packet.count; uint32_t bit = 1 << id; int pos = decoder->blk_type_pos;
if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { intel_pt_log("WARNING: Unknown block item %u type %d\n", id, decoder->blk_type); return; }
if (decoder->state.items.mask[pos] & bit) { intel_pt_log("WARNING: Duplicate block item %u type %d\n", id, decoder->blk_type); }
decoder->state.items.mask[pos] |= bit; decoder->state.items.val[pos][id] = decoder->packet.payload;}
/* Walk PSB+ packets when already in sync. */static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder){ int err;
decoder->in_psb = true;
while (1) { err = intel_pt_get_next_packet(decoder); if (err) goto out;
switch (decoder->packet.type) { case INTEL_PT_PSBEND: err = 0; goto out;
case INTEL_PT_TIP_PGD: case INTEL_PT_TIP_PGE: case INTEL_PT_TIP: case INTEL_PT_TNT: case INTEL_PT_TRACESTOP: case INTEL_PT_BAD: case INTEL_PT_PSB: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: decoder->have_tma = false; intel_pt_log("ERROR: Unexpected packet\n"); err = -EAGAIN; goto out;
case INTEL_PT_OVF: err = intel_pt_overflow(decoder); goto out;
case INTEL_PT_TSC: intel_pt_calc_tsc_timestamp(decoder); break;
case INTEL_PT_TMA: intel_pt_calc_tma(decoder); break;
case INTEL_PT_CBR: intel_pt_calc_cbr(decoder); break;
case INTEL_PT_MODE_EXEC: intel_pt_mode_exec_status(decoder); break;
case INTEL_PT_PIP: intel_pt_set_pip(decoder); break;
case INTEL_PT_FUP: decoder->pge = true; if (decoder->packet.count) { intel_pt_set_last_ip(decoder); decoder->psb_ip = decoder->last_ip; } break;
case INTEL_PT_MODE_TSX: intel_pt_update_in_tx(decoder); break;
case INTEL_PT_MTC: intel_pt_calc_mtc_timestamp(decoder); if (decoder->period_type == INTEL_PT_PERIOD_MTC) decoder->state.type |= INTEL_PT_INSTRUCTION; break;
case INTEL_PT_CYC: intel_pt_calc_cyc_timestamp(decoder); break;
case INTEL_PT_VMCS: case INTEL_PT_MNT: case INTEL_PT_PAD: default: break; } }out: decoder->in_psb = false;
return err;}
static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder){ int err;
if (decoder->tx_flags & INTEL_PT_ABORT_TX) { decoder->tx_flags = 0; decoder->state.flags &= ~INTEL_PT_IN_TX; decoder->state.flags |= INTEL_PT_ABORT_TX; } else { decoder->state.flags |= INTEL_PT_ASYNC; }
while (1) { err = intel_pt_get_next_packet(decoder); if (err) return err;
switch (decoder->packet.type) { case INTEL_PT_TNT: case INTEL_PT_FUP: case INTEL_PT_TRACESTOP: case INTEL_PT_PSB: case INTEL_PT_TSC: case INTEL_PT_TMA: case INTEL_PT_MODE_TSX: case INTEL_PT_BAD: case INTEL_PT_PSBEND: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: intel_pt_log("ERROR: Missing TIP after FUP\n"); decoder->pkt_state = INTEL_PT_STATE_ERR3; decoder->pkt_step = 0; return -ENOENT;
case INTEL_PT_CBR: intel_pt_calc_cbr(decoder); break;
case INTEL_PT_OVF: return intel_pt_overflow(decoder);
case INTEL_PT_TIP_PGD: decoder->state.from_ip = decoder->ip; if (decoder->packet.count == 0) { decoder->state.to_ip = 0; } else { intel_pt_set_ip(decoder); decoder->state.to_ip = decoder->ip; } decoder->pge = false; decoder->continuous_period = false; decoder->state.type |= INTEL_PT_TRACE_END; intel_pt_update_nr(decoder); return 0;
case INTEL_PT_TIP_PGE: decoder->pge = true; intel_pt_log("Omitting PGE ip " x64_fmt "\n", decoder->ip); decoder->state.from_ip = 0; if (decoder->packet.count == 0) { decoder->state.to_ip = 0; } else { intel_pt_set_ip(decoder); decoder->state.to_ip = decoder->ip; } decoder->state.type |= INTEL_PT_TRACE_BEGIN; intel_pt_mtc_cyc_cnt_pge(decoder); intel_pt_set_nr(decoder); return 0;
case INTEL_PT_TIP: decoder->state.from_ip = decoder->ip; if (decoder->packet.count == 0) { decoder->state.to_ip = 0; } else { intel_pt_set_ip(decoder); decoder->state.to_ip = decoder->ip; } intel_pt_update_nr(decoder); intel_pt_sample_iflag_chg(decoder); return 0;
case INTEL_PT_PIP: intel_pt_update_pip(decoder); break;
case INTEL_PT_MTC: intel_pt_calc_mtc_timestamp(decoder); if (decoder->period_type == INTEL_PT_PERIOD_MTC) decoder->state.type |= INTEL_PT_INSTRUCTION; break;
case INTEL_PT_CYC: intel_pt_calc_cyc_timestamp(decoder); break;
case INTEL_PT_MODE_EXEC: intel_pt_mode_exec(decoder); break;
case INTEL_PT_VMCS: case INTEL_PT_MNT: case INTEL_PT_PAD: break;
default: return intel_pt_bug(decoder); } }}
static int intel_pt_resample(struct intel_pt_decoder *decoder){ decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->state.type = INTEL_PT_INSTRUCTION; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; return 0;}
struct intel_pt_vm_tsc_info { struct intel_pt_pkt pip_packet; struct intel_pt_pkt vmcs_packet; struct intel_pt_pkt tma_packet; bool tsc, pip, vmcs, tma, psbend; uint64_t ctc_delta; uint64_t last_ctc; int max_lookahead;};
/* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info){ struct intel_pt_vm_tsc_info *data = pkt_info->data;
switch (pkt_info->packet.type) { case INTEL_PT_PAD: case INTEL_PT_MNT: case INTEL_PT_MODE_EXEC: case INTEL_PT_MODE_TSX: case INTEL_PT_MTC: case INTEL_PT_FUP: case INTEL_PT_CYC: case INTEL_PT_CBR: break;
case INTEL_PT_TSC: data->tsc = true; break;
case INTEL_PT_TMA: data->tma_packet = pkt_info->packet; data->tma = true; break;
case INTEL_PT_PIP: data->pip_packet = pkt_info->packet; data->pip = true; break;
case INTEL_PT_VMCS: data->vmcs_packet = pkt_info->packet; data->vmcs = true; break;
case INTEL_PT_PSBEND: data->psbend = true; return 1;
case INTEL_PT_TIP_PGE: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_OVF: case INTEL_PT_BAD: case INTEL_PT_TNT: case INTEL_PT_TIP_PGD: case INTEL_PT_TIP: case INTEL_PT_PSB: case INTEL_PT_TRACESTOP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: default: return 1; }
return 0;}
struct intel_pt_ovf_fup_info { int max_lookahead; bool found;};
/* Lookahead to detect a FUP packet after OVF */static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info){ struct intel_pt_ovf_fup_info *data = pkt_info->data;
if (pkt_info->packet.type == INTEL_PT_CYC || pkt_info->packet.type == INTEL_PT_MTC || pkt_info->packet.type == INTEL_PT_TSC) return !--(data->max_lookahead); data->found = pkt_info->packet.type == INTEL_PT_FUP; return 1;}
static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder){ struct intel_pt_ovf_fup_info data = { .max_lookahead = 16, .found = false, };
intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); return data.found;}
/* Lookahead and get the TMA packet after TSC */static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info){ struct intel_pt_vm_tsc_info *data = pkt_info->data;
if (pkt_info->packet.type == INTEL_PT_CYC || pkt_info->packet.type == INTEL_PT_MTC) return !--(data->max_lookahead);
if (pkt_info->packet.type == INTEL_PT_TMA) { data->tma_packet = pkt_info->packet; data->tma = true; } return 1;}
static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc){ if (decoder->tsc_ctc_mult) return ctc * decoder->tsc_ctc_mult; else return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);}
static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, uint32_t ctc, uint32_t fc, uint64_t last_ctc_timestamp, uint64_t ctc_delta, uint32_t last_ctc){ /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ uint64_t last_mtc_ctc = last_ctc + ctc_delta; /* * Number of CTC ticks from there until current TMA packet. We would * expect last_mtc_ctc to be before ctc, but the TSC packet can slip * past an MTC, so a sign-extended value is used. */ uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ uint64_t new_ctc_delta = ctc_delta + delta; uint64_t expected_tsc;
/* * Convert CTC ticks to TSC ticks, add the starting point * (last_ctc_timestamp) and the fast counter from the TMA packet. */ expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
if (intel_pt_enable_logging) { intel_pt_log_x64(last_mtc_ctc); intel_pt_log_x32(last_ctc); intel_pt_log_x64(ctc_delta); intel_pt_log_x64(delta); intel_pt_log_x32(ctc); intel_pt_log_x64(new_ctc_delta); intel_pt_log_x64(last_ctc_timestamp); intel_pt_log_x32(fc); intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); intel_pt_log_x64(expected_tsc); }
return expected_tsc;}
static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, struct intel_pt_vm_tsc_info *data){ uint32_t ctc = data->tma_packet.payload; uint32_t fc = data->tma_packet.count;
return intel_pt_calc_expected_tsc(decoder, ctc, fc, decoder->ctc_timestamp, data->ctc_delta, data->last_ctc);}
static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, struct intel_pt_vmcs_info *vmcs_info){ uint64_t payload = decoder->packet.payload;
/* VMX adds the TSC Offset, so subtract to get host TSC */ decoder->packet.payload -= vmcs_info->tsc_offset; /* TSC packet has only 7 bytes */ decoder->packet.payload &= SEVEN_BYTES;
/* * The buffer is mmapped from the data file, so this also updates the * data file. */ if (!decoder->vm_tm_corr_dry_run) memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", payload, decoder->packet.payload, vmcs_info->vmcs, vmcs_info->tsc_offset);}
static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, uint64_t tsc_offset){ struct intel_pt_vmcs_info vmcs_info = { .vmcs = NO_VMCS, .tsc_offset = tsc_offset };
intel_pt_translate_vm_tsc(decoder, &vmcs_info);}
static inline bool in_vm(uint64_t pip_payload){ return pip_payload & 1;}
static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet){ return pip_packet->payload & 1;}
static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info){ p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, vmcs_info->vmcs, vmcs_info->tsc_offset);}
static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, struct intel_pt_vm_tsc_info *data){ memset(data, 0, sizeof(*data)); data->ctc_delta = decoder->ctc_delta; data->last_ctc = decoder->last_ctc; intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); if (data->tsc && !data->psbend) p_log("ERROR: PSB without PSBEND"); decoder->in_psb = data->psbend;}
static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, struct intel_pt_vm_tsc_info *data, struct intel_pt_vmcs_info *vmcs_info, uint64_t host_tsc){ if (!decoder->in_psb) { /* Can't happen */ p_log("ERROR: First TSC is not in PSB+"); }
if (data->pip) { if (pip_in_vm(&data->pip_packet)) { /* Guest */ if (vmcs_info && vmcs_info->tsc_offset) { intel_pt_translate_vm_tsc(decoder, vmcs_info); decoder->vm_tm_corr_reliable = true; } else { p_log("ERROR: First TSC, unknown TSC Offset"); } } else { /* Host */ decoder->vm_tm_corr_reliable = true; } } else { /* Host or Guest */ decoder->vm_tm_corr_reliable = false; if (intel_pt_time_in_range(decoder, host_tsc)) { /* Assume Host */ } else { /* Assume Guest */ if (vmcs_info && vmcs_info->tsc_offset) intel_pt_translate_vm_tsc(decoder, vmcs_info); else p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); } }}
static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, struct intel_pt_vm_tsc_info *data){ struct intel_pt_vmcs_info *vmcs_info; uint64_t tsc_offset = 0; uint64_t vmcs; bool reliable = true; uint64_t expected_tsc; uint64_t host_tsc; uint64_t ref_timestamp;
bool assign = false; bool assign_reliable = false;
/* Already have 'data' for the in_psb case */ if (!decoder->in_psb) { memset(data, 0, sizeof(*data)); data->ctc_delta = decoder->ctc_delta; data->last_ctc = decoder->last_ctc; data->max_lookahead = 16; intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); if (decoder->pge) { data->pip = true; data->pip_packet.payload = decoder->pip_payload; } }
/* Calculations depend on having TMA packets */ if (!data->tma) { p_log("ERROR: TSC without TMA"); return; }
vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; if (vmcs == NO_VMCS) vmcs = 0;
vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
if (!decoder->ctc_timestamp) { intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); return; }
expected_tsc = intel_pt_expected_tsc(decoder, data);
tsc_offset = host_tsc - expected_tsc;
/* Determine if TSC is from Host or Guest */ if (data->pip) { if (pip_in_vm(&data->pip_packet)) { /* Guest */ if (!vmcs_info) { /* PIP NR=1 without VMCS cannot happen */ p_log("ERROR: Missing VMCS"); intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); decoder->vm_tm_corr_reliable = false; return; } } else { /* Host */ decoder->last_reliable_timestamp = host_tsc; decoder->vm_tm_corr_reliable = true; return; } } else { /* Host or Guest */ reliable = false; /* Host/Guest is a guess, so not reliable */ if (decoder->in_psb) { if (!tsc_offset) return; /* Zero TSC Offset, assume Host */ /* * TSC packet has only 7 bytes of TSC. We have no * information about the Guest's 8th byte, but it * doesn't matter because we only need 7 bytes. * Here, since the 8th byte is unreliable and * irrelevant, compare only 7 byes. */ if (vmcs_info && (tsc_offset & SEVEN_BYTES) == (vmcs_info->tsc_offset & SEVEN_BYTES)) { /* Same TSC Offset as last VMCS, assume Guest */ goto guest; } } /* * Check if the host_tsc is within the expected range. * Note, we could narrow the range more by looking ahead for * the next host TSC in the same buffer, but we don't bother to * do that because this is probably good enough. */ if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { /* Within expected range for Host TSC, assume Host */ decoder->vm_tm_corr_reliable = false; return; } }
guest: /* Assuming Guest */
/* Determine whether to assign TSC Offset */ if (vmcs_info && vmcs_info->vmcs) { if (vmcs_info->tsc_offset && vmcs_info->reliable) { assign = false; } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { /* Continuous tracing, TSC in a PSB is not a time loss */ assign = true; assign_reliable = true; } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { /* * Unlikely to be a time loss TSC in a PSB which is not * at the start of a buffer. */ assign = true; assign_reliable = false; } }
/* Record VMCS TSC Offset */ if (assign && (vmcs_info->tsc_offset != tsc_offset || vmcs_info->reliable != assign_reliable)) { bool print = vmcs_info->tsc_offset != tsc_offset;
vmcs_info->tsc_offset = tsc_offset; vmcs_info->reliable = assign_reliable; if (print) intel_pt_print_vmcs_info(vmcs_info); }
/* Determine what TSC Offset to use */ if (vmcs_info && vmcs_info->tsc_offset) { if (!vmcs_info->reliable) reliable = false; intel_pt_translate_vm_tsc(decoder, vmcs_info); } else { reliable = false; if (vmcs_info) { if (!vmcs_info->error_printed) { p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, vmcs_info->vmcs); vmcs_info->error_printed = true; } } else { if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) p_log("ERROR: Unknown VMCS"); } intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); }
decoder->vm_tm_corr_reliable = reliable;}
static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder){ uint64_t host_tsc = decoder->packet.payload; uint64_t guest_tsc = decoder->packet.payload; struct intel_pt_vmcs_info *vmcs_info; uint64_t vmcs;
vmcs = decoder->vmcs; if (vmcs == NO_VMCS) vmcs = 0;
vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
if (decoder->pge) { if (in_vm(decoder->pip_payload)) { /* Guest */ if (!vmcs_info) { /* PIP NR=1 without VMCS cannot happen */ p_log("ERROR: Missing VMCS"); } } else { /* Host */ return; } } else { /* Host or Guest */ if (intel_pt_time_in_range(decoder, host_tsc)) { /* Within expected range for Host TSC, assume Host */ return; } }
if (vmcs_info) { /* Translate Guest TSC to Host TSC */ host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", guest_tsc, host_tsc, vmcs_info->vmcs, vmcs_info->tsc_offset); if (!intel_pt_time_in_range(decoder, host_tsc) && intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) p_log("Timestamp out of range"); } else { if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) p_log("ERROR: Unknown VMCS"); host_tsc = decoder->timestamp; }
decoder->packet.payload = host_tsc;
if (!decoder->vm_tm_corr_dry_run) memcpy((void *)decoder->buf + 1, &host_tsc, 8);}
static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder){ struct intel_pt_vm_tsc_info data = { .psbend = false }; bool pge; int err;
if (decoder->in_psb) intel_pt_vm_tm_corr_psb(decoder, &data);
while (1) { err = intel_pt_get_next_packet(decoder); if (err == -ENOLINK) continue; if (err) break;
switch (decoder->packet.type) { case INTEL_PT_TIP_PGD: decoder->pge = false; decoder->vm_tm_corr_continuous = false; break;
case INTEL_PT_TNT: case INTEL_PT_TIP: case INTEL_PT_TIP_PGE: decoder->pge = true; break;
case INTEL_PT_OVF: decoder->in_psb = false; pge = decoder->pge; decoder->pge = intel_pt_ovf_fup_lookahead(decoder); if (pge != decoder->pge) intel_pt_log("Surprising PGE change in OVF!"); if (!decoder->pge) decoder->vm_tm_corr_continuous = false; break;
case INTEL_PT_FUP: if (decoder->in_psb) decoder->pge = true; break;
case INTEL_PT_TRACESTOP: decoder->pge = false; decoder->vm_tm_corr_continuous = false; decoder->have_tma = false; break;
case INTEL_PT_PSB: intel_pt_vm_tm_corr_psb(decoder, &data); break;
case INTEL_PT_PIP: decoder->pip_payload = decoder->packet.payload; break;
case INTEL_PT_MTC: intel_pt_calc_mtc_timestamp(decoder); break;
case INTEL_PT_TSC: intel_pt_vm_tm_corr_tsc(decoder, &data); intel_pt_calc_tsc_timestamp(decoder); decoder->vm_tm_corr_same_buf = true; decoder->vm_tm_corr_continuous = decoder->pge; break;
case INTEL_PT_TMA: intel_pt_calc_tma(decoder); break;
case INTEL_PT_CYC: intel_pt_calc_cyc_timestamp(decoder); break;
case INTEL_PT_CBR: intel_pt_calc_cbr(decoder); break;
case INTEL_PT_PSBEND: decoder->in_psb = false; data.psbend = false; break;
case INTEL_PT_VMCS: if (decoder->packet.payload != NO_VMCS) decoder->vmcs = decoder->packet.payload; break;
case INTEL_PT_BBP: decoder->blk_type = decoder->packet.payload; break;
case INTEL_PT_BIP: if (decoder->blk_type == INTEL_PT_PEBS_BASIC && decoder->packet.count == 2) intel_pt_vm_tm_corr_pebs_tsc(decoder); break;
case INTEL_PT_BEP: case INTEL_PT_BEP_IP: decoder->blk_type = 0; break;
case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: case INTEL_PT_MODE_EXEC: case INTEL_PT_MODE_TSX: case INTEL_PT_MNT: case INTEL_PT_PAD: case INTEL_PT_PTWRITE_IP: case INTEL_PT_PTWRITE: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_EXSTOP_IP: case INTEL_PT_EXSTOP: case INTEL_PT_PWRX: case INTEL_PT_BAD: /* Does not happen */ default: break; } }
return err;}
#define HOP_PROCESS 0#define HOP_IGNORE 1#define HOP_RETURN 2#define HOP_AGAIN 3
static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err){ *err = 0;
/* Leap from PSB to PSB, getting ip from FUP within PSB+ */ if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { *err = intel_pt_scan_for_psb(decoder); if (*err) return HOP_RETURN; }
switch (decoder->packet.type) { case INTEL_PT_TNT: return HOP_IGNORE;
case INTEL_PT_TIP_PGD: decoder->pge = false; if (!decoder->packet.count) { intel_pt_set_nr(decoder); return HOP_IGNORE; } intel_pt_set_ip(decoder); decoder->state.type |= INTEL_PT_TRACE_END; decoder->state.from_ip = 0; decoder->state.to_ip = decoder->ip; intel_pt_update_nr(decoder); return HOP_RETURN;
case INTEL_PT_TIP: if (!decoder->packet.count) { intel_pt_set_nr(decoder); return HOP_IGNORE; } intel_pt_set_ip(decoder); decoder->state.type = INTEL_PT_INSTRUCTION; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; intel_pt_update_nr(decoder); intel_pt_sample_iflag_chg(decoder); return HOP_RETURN;
case INTEL_PT_FUP: if (!decoder->packet.count) return HOP_IGNORE; intel_pt_set_ip(decoder); if (decoder->set_fup_mwait || decoder->set_fup_pwre) *no_tip = true; if (!decoder->branch_enable || !decoder->pge) *no_tip = true; if (*no_tip) { decoder->state.type = INTEL_PT_INSTRUCTION; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; intel_pt_fup_event(decoder, *no_tip); return HOP_RETURN; } intel_pt_fup_event(decoder, *no_tip); decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; *err = intel_pt_walk_fup_tip(decoder); if (!*err && decoder->state.to_ip) decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; return HOP_RETURN;
case INTEL_PT_PSB: decoder->state.psb_offset = decoder->pos; decoder->psb_ip = 0; decoder->last_ip = 0; decoder->have_last_ip = true; *err = intel_pt_walk_psbend(decoder); if (*err == -EAGAIN) return HOP_AGAIN; if (*err) return HOP_RETURN; decoder->state.type = INTEL_PT_PSB_EVT; if (decoder->psb_ip) { decoder->state.type |= INTEL_PT_INSTRUCTION; decoder->ip = decoder->psb_ip; } decoder->state.from_ip = decoder->psb_ip; decoder->state.to_ip = 0; return HOP_RETURN;
case INTEL_PT_BAD: case INTEL_PT_PAD: case INTEL_PT_TIP_PGE: case INTEL_PT_TSC: case INTEL_PT_TMA: case INTEL_PT_MODE_EXEC: case INTEL_PT_MODE_TSX: case INTEL_PT_MTC: case INTEL_PT_CYC: case INTEL_PT_VMCS: case INTEL_PT_PSBEND: case INTEL_PT_CBR: case INTEL_PT_TRACESTOP: case INTEL_PT_PIP: case INTEL_PT_OVF: case INTEL_PT_MNT: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: default: return HOP_PROCESS; }}
struct intel_pt_psb_info { struct intel_pt_pkt fup_packet; bool fup; int after_psbend;};
/* Lookahead and get the FUP packet from PSB+ */static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info){ struct intel_pt_psb_info *data = pkt_info->data;
switch (pkt_info->packet.type) { case INTEL_PT_PAD: case INTEL_PT_MNT: case INTEL_PT_TSC: case INTEL_PT_TMA: case INTEL_PT_MODE_EXEC: case INTEL_PT_MODE_TSX: case INTEL_PT_MTC: case INTEL_PT_CYC: case INTEL_PT_VMCS: case INTEL_PT_CBR: case INTEL_PT_PIP: if (data->after_psbend) { data->after_psbend -= 1; if (!data->after_psbend) return 1; } break;
case INTEL_PT_FUP: if (data->after_psbend) return 1; if (data->fup || pkt_info->packet.count == 0) return 1; data->fup_packet = pkt_info->packet; data->fup = true; break;
case INTEL_PT_PSBEND: if (!data->fup) return 1; /* Keep going to check for a TIP.PGE */ data->after_psbend = 6; break;
case INTEL_PT_TIP_PGE: /* Ignore FUP in PSB+ if followed by TIP.PGE */ if (data->after_psbend) data->fup = false; return 1;
case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: if (data->after_psbend) { data->after_psbend -= 1; if (!data->after_psbend) return 1; break; } return 1;
case INTEL_PT_OVF: case INTEL_PT_BAD: case INTEL_PT_TNT: case INTEL_PT_TIP_PGD: case INTEL_PT_TIP: case INTEL_PT_PSB: case INTEL_PT_TRACESTOP: default: return 1; }
return 0;}
static int intel_pt_psb(struct intel_pt_decoder *decoder){ int err;
decoder->last_ip = 0; decoder->psb_ip = 0; decoder->have_last_ip = true; intel_pt_clear_stack(&decoder->stack); err = intel_pt_walk_psbend(decoder); if (err) return err; decoder->state.type = INTEL_PT_PSB_EVT; decoder->state.from_ip = decoder->psb_ip; decoder->state.to_ip = 0; return 0;}
static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder){ int err;
if (decoder->ip != decoder->last_ip) { err = intel_pt_walk_fup(decoder); if (!err || err != -EAGAIN) return err; }
decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; err = intel_pt_psb(decoder); if (err) { decoder->pkt_state = INTEL_PT_STATE_ERR3; return -ENOENT; }
return 0;}
static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err){ struct intel_pt_psb_info data = { .fup = false };
if (!decoder->branch_enable) return false;
intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); if (!data.fup) return false;
decoder->packet = data.fup_packet; intel_pt_set_last_ip(decoder); decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
*err = intel_pt_fup_in_psb(decoder);
return true;}
static int intel_pt_walk_trace(struct intel_pt_decoder *decoder){ int last_packet_type = INTEL_PT_PAD; bool no_tip = false; int err;
while (1) { err = intel_pt_get_next_packet(decoder); if (err) return err;next: err = 0; if (decoder->cyc_threshold) { if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) decoder->sample_cyc = false; last_packet_type = decoder->packet.type; }
if (decoder->hop) { switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { case HOP_IGNORE: continue; case HOP_RETURN: return err; case HOP_AGAIN: goto next; default: break; } }
switch (decoder->packet.type) { case INTEL_PT_TNT: if (!decoder->packet.count) break; decoder->tnt = decoder->packet; decoder->pkt_state = INTEL_PT_STATE_TNT; err = intel_pt_walk_tnt(decoder); if (err == -EAGAIN) break; return err;
case INTEL_PT_TIP_PGD: if (decoder->packet.count != 0) intel_pt_set_last_ip(decoder); decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; return intel_pt_walk_tip(decoder);
case INTEL_PT_TIP_PGE: { decoder->pge = true; decoder->overflow = false; intel_pt_mtc_cyc_cnt_pge(decoder); intel_pt_set_nr(decoder); if (decoder->packet.count == 0) { intel_pt_log_at("Skipping zero TIP.PGE", decoder->pos); break; } intel_pt_sample_iflag_chg(decoder); intel_pt_set_ip(decoder); decoder->state.from_ip = 0; decoder->state.to_ip = decoder->ip; decoder->state.type |= INTEL_PT_TRACE_BEGIN; /* * In hop mode, resample to get the to_ip as an * "instruction" sample. */ if (decoder->hop) decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; return 0; }
case INTEL_PT_OVF: return intel_pt_overflow(decoder);
case INTEL_PT_TIP: if (decoder->packet.count != 0) intel_pt_set_last_ip(decoder); decoder->pkt_state = INTEL_PT_STATE_TIP; return intel_pt_walk_tip(decoder);
case INTEL_PT_FUP: if (decoder->packet.count == 0) { intel_pt_log_at("Skipping zero FUP", decoder->pos); no_tip = false; break; } intel_pt_set_last_ip(decoder); if (!decoder->branch_enable || !decoder->pge) { decoder->ip = decoder->last_ip; if (intel_pt_fup_event(decoder, no_tip)) return 0; no_tip = false; break; } if (decoder->set_fup_mwait) no_tip = true; if (no_tip) decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; else decoder->pkt_state = INTEL_PT_STATE_FUP; err = intel_pt_walk_fup(decoder); if (err != -EAGAIN) return err; if (no_tip) { no_tip = false; break; } return intel_pt_walk_fup_tip(decoder);
case INTEL_PT_TRACESTOP: decoder->pge = false; decoder->continuous_period = false; intel_pt_clear_tx_flags(decoder); decoder->have_tma = false; break;
case INTEL_PT_PSB: decoder->state.psb_offset = decoder->pos; decoder->psb_ip = 0; if (intel_pt_psb_with_fup(decoder, &err)) return err; err = intel_pt_psb(decoder); if (err == -EAGAIN) goto next; return err;
case INTEL_PT_PIP: intel_pt_update_pip(decoder); break;
case INTEL_PT_MTC: intel_pt_calc_mtc_timestamp(decoder); if (decoder->period_type != INTEL_PT_PERIOD_MTC) break; /* * Ensure that there has been an instruction since the * last MTC. */ if (!decoder->mtc_insn) break; decoder->mtc_insn = false; /* Ensure that there is a timestamp */ if (!decoder->timestamp) break; decoder->state.type = INTEL_PT_INSTRUCTION; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->mtc_insn = false; return 0;
case INTEL_PT_TSC: intel_pt_calc_tsc_timestamp(decoder); break;
case INTEL_PT_TMA: intel_pt_calc_tma(decoder); break;
case INTEL_PT_CYC: intel_pt_calc_cyc_timestamp(decoder); break;
case INTEL_PT_CBR: intel_pt_calc_cbr(decoder); if (decoder->cbr != decoder->cbr_seen) { decoder->state.type = 0; return 0; } break;
case INTEL_PT_MODE_EXEC: intel_pt_mode_exec(decoder); err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type == INTEL_PT_FUP) { decoder->set_fup_mode_exec = true; no_tip = true; } goto next;
case INTEL_PT_MODE_TSX: /* MODE_TSX need not be followed by FUP */ if (!decoder->pge || decoder->in_psb) { intel_pt_update_in_tx(decoder); break; } err = intel_pt_mode_tsx(decoder, &no_tip); if (err) return err; goto next;
case INTEL_PT_BAD: /* Does not happen */ return intel_pt_bug(decoder);
case INTEL_PT_PSBEND: case INTEL_PT_VMCS: case INTEL_PT_MNT: case INTEL_PT_PAD: break;
case INTEL_PT_PTWRITE_IP: decoder->fup_ptw_payload = decoder->packet.payload; err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type == INTEL_PT_FUP) { decoder->set_fup_ptw = true; no_tip = true; } else { intel_pt_log_at("ERROR: Missing FUP after PTWRITE", decoder->pos); } goto next;
case INTEL_PT_PTWRITE: decoder->state.type = INTEL_PT_PTW; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->state.ptw_payload = decoder->packet.payload; return 0;
case INTEL_PT_MWAIT: decoder->fup_mwait_payload = decoder->packet.payload; decoder->set_fup_mwait = true; break;
case INTEL_PT_PWRE: if (decoder->set_fup_mwait) { decoder->fup_pwre_payload = decoder->packet.payload; decoder->set_fup_pwre = true; break; } decoder->state.type = INTEL_PT_PWR_ENTRY; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->state.pwrx_payload = decoder->packet.payload; return 0;
case INTEL_PT_EXSTOP_IP: err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type == INTEL_PT_FUP) { decoder->set_fup_exstop = true; no_tip = true; } else { intel_pt_log_at("ERROR: Missing FUP after EXSTOP", decoder->pos); } goto next;
case INTEL_PT_EXSTOP: decoder->state.type = INTEL_PT_EX_STOP; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; return 0;
case INTEL_PT_PWRX: decoder->state.type = INTEL_PT_PWR_EXIT; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->state.pwrx_payload = decoder->packet.payload; return 0;
case INTEL_PT_BBP: intel_pt_bbp(decoder); break;
case INTEL_PT_BIP: intel_pt_bip(decoder); break;
case INTEL_PT_BEP: decoder->state.type = INTEL_PT_BLK_ITEMS; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; return 0;
case INTEL_PT_BEP_IP: err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type == INTEL_PT_FUP) { decoder->set_fup_bep = true; no_tip = true; } else { intel_pt_log_at("ERROR: Missing FUP after BEP", decoder->pos); } goto next;
case INTEL_PT_CFE: decoder->fup_cfe_pkt = decoder->packet; decoder->set_fup_cfe = true; if (!decoder->pge) { intel_pt_fup_event(decoder, true); return 0; } break;
case INTEL_PT_CFE_IP: decoder->fup_cfe_pkt = decoder->packet; err = intel_pt_get_next_packet(decoder); if (err) return err; if (decoder->packet.type == INTEL_PT_FUP) { decoder->set_fup_cfe_ip = true; no_tip = true; } else { intel_pt_log_at("ERROR: Missing FUP after CFE", decoder->pos); } goto next;
case INTEL_PT_EVD: err = intel_pt_evd(decoder); if (err) return err; break;
default: return intel_pt_bug(decoder); } }}
static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder){ return decoder->packet.count && (decoder->have_last_ip || decoder->packet.count == 3 || decoder->packet.count == 6);}
/* Walk PSB+ packets to get in sync. */static int intel_pt_walk_psb(struct intel_pt_decoder *decoder){ int err;
decoder->in_psb = true;
while (1) { err = intel_pt_get_next_packet(decoder); if (err) goto out;
switch (decoder->packet.type) { case INTEL_PT_TIP_PGD: decoder->continuous_period = false; fallthrough; case INTEL_PT_TIP_PGE: case INTEL_PT_TIP: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: intel_pt_log("ERROR: Unexpected packet\n"); err = -ENOENT; goto out;
case INTEL_PT_FUP: decoder->pge = true; if (intel_pt_have_ip(decoder)) { uint64_t current_ip = decoder->ip;
intel_pt_set_ip(decoder); decoder->psb_ip = decoder->ip; if (current_ip) intel_pt_log_to("Setting IP", decoder->ip); } break;
case INTEL_PT_MTC: intel_pt_calc_mtc_timestamp(decoder); break;
case INTEL_PT_TSC: intel_pt_calc_tsc_timestamp(decoder); break;
case INTEL_PT_TMA: intel_pt_calc_tma(decoder); break;
case INTEL_PT_CYC: intel_pt_calc_cyc_timestamp(decoder); break;
case INTEL_PT_CBR: intel_pt_calc_cbr(decoder); break;
case INTEL_PT_PIP: intel_pt_set_pip(decoder); break;
case INTEL_PT_MODE_EXEC: intel_pt_mode_exec_status(decoder); break;
case INTEL_PT_MODE_TSX: intel_pt_update_in_tx(decoder); break;
case INTEL_PT_TRACESTOP: decoder->pge = false; decoder->continuous_period = false; intel_pt_clear_tx_flags(decoder); fallthrough;
case INTEL_PT_TNT: decoder->have_tma = false; intel_pt_log("ERROR: Unexpected packet\n"); if (decoder->ip) decoder->pkt_state = INTEL_PT_STATE_ERR4; else decoder->pkt_state = INTEL_PT_STATE_ERR3; err = -ENOENT; goto out;
case INTEL_PT_BAD: /* Does not happen */ err = intel_pt_bug(decoder); goto out;
case INTEL_PT_OVF: err = intel_pt_overflow(decoder); goto out;
case INTEL_PT_PSBEND: err = 0; goto out;
case INTEL_PT_PSB: case INTEL_PT_VMCS: case INTEL_PT_MNT: case INTEL_PT_PAD: default: break; } }out: decoder->in_psb = false;
return err;}
static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder){ int err;
while (1) { err = intel_pt_get_next_packet(decoder); if (err) return err;
switch (decoder->packet.type) { case INTEL_PT_TIP_PGD: decoder->continuous_period = false; decoder->pge = false; if (intel_pt_have_ip(decoder)) intel_pt_set_ip(decoder); if (!decoder->ip) break; decoder->state.type |= INTEL_PT_TRACE_END; return 0;
case INTEL_PT_TIP_PGE: decoder->pge = true; intel_pt_mtc_cyc_cnt_pge(decoder); if (intel_pt_have_ip(decoder)) intel_pt_set_ip(decoder); if (!decoder->ip) break; decoder->state.type |= INTEL_PT_TRACE_BEGIN; return 0;
case INTEL_PT_TIP: decoder->pge = true; if (intel_pt_have_ip(decoder)) intel_pt_set_ip(decoder); if (!decoder->ip) break; return 0;
case INTEL_PT_FUP: if (intel_pt_have_ip(decoder)) intel_pt_set_ip(decoder); if (decoder->ip) return 0; break;
case INTEL_PT_MTC: intel_pt_calc_mtc_timestamp(decoder); break;
case INTEL_PT_TSC: intel_pt_calc_tsc_timestamp(decoder); break;
case INTEL_PT_TMA: intel_pt_calc_tma(decoder); break;
case INTEL_PT_CYC: intel_pt_calc_cyc_timestamp(decoder); break;
case INTEL_PT_CBR: intel_pt_calc_cbr(decoder); break;
case INTEL_PT_PIP: intel_pt_set_pip(decoder); break;
case INTEL_PT_MODE_EXEC: intel_pt_mode_exec_status(decoder); break;
case INTEL_PT_MODE_TSX: intel_pt_update_in_tx(decoder); break;
case INTEL_PT_OVF: return intel_pt_overflow(decoder);
case INTEL_PT_BAD: /* Does not happen */ return intel_pt_bug(decoder);
case INTEL_PT_TRACESTOP: decoder->pge = false; decoder->continuous_period = false; intel_pt_clear_tx_flags(decoder); decoder->have_tma = false; break;
case INTEL_PT_PSB: decoder->state.psb_offset = decoder->pos; decoder->psb_ip = 0; decoder->last_ip = 0; decoder->have_last_ip = true; intel_pt_clear_stack(&decoder->stack); err = intel_pt_walk_psb(decoder); if (err) return err; decoder->state.type = INTEL_PT_PSB_EVT; decoder->state.from_ip = decoder->psb_ip; decoder->state.to_ip = 0; return 0;
case INTEL_PT_TNT: case INTEL_PT_PSBEND: case INTEL_PT_VMCS: case INTEL_PT_MNT: case INTEL_PT_PAD: case INTEL_PT_PTWRITE: case INTEL_PT_PTWRITE_IP: case INTEL_PT_EXSTOP: case INTEL_PT_EXSTOP_IP: case INTEL_PT_MWAIT: case INTEL_PT_PWRE: case INTEL_PT_PWRX: case INTEL_PT_BBP: case INTEL_PT_BIP: case INTEL_PT_BEP: case INTEL_PT_BEP_IP: case INTEL_PT_CFE: case INTEL_PT_CFE_IP: case INTEL_PT_EVD: default: break; } }}
static int intel_pt_sync_ip(struct intel_pt_decoder *decoder){ int err;
intel_pt_clear_fup_event(decoder); decoder->overflow = false;
if (!decoder->branch_enable) { decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; decoder->state.type = 0; /* Do not have a sample */ return 0; }
intel_pt_log("Scanning for full IP\n"); err = intel_pt_walk_to_ip(decoder); if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) return err;
/* In hop mode, resample to get the to_ip as an "instruction" sample */ if (decoder->hop) decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; else decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
decoder->state.from_ip = 0; decoder->state.to_ip = decoder->ip; intel_pt_log_to("Setting IP", decoder->ip);
return 0;}
static int intel_pt_part_psb(struct intel_pt_decoder *decoder){ const unsigned char *end = decoder->buf + decoder->len; size_t i;
for (i = INTEL_PT_PSB_LEN - 1; i; i--) { if (i > decoder->len) continue; if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) return i; } return 0;}
static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb){ size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; const char *psb = INTEL_PT_PSB_STR;
if (rest_psb > decoder->len || memcmp(decoder->buf, psb + part_psb, rest_psb)) return 0;
return rest_psb;}
static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, int part_psb){ int rest_psb, ret;
decoder->pos += decoder->len; decoder->len = 0;
ret = intel_pt_get_next_data(decoder, false); if (ret) return ret;
rest_psb = intel_pt_rest_psb(decoder, part_psb); if (!rest_psb) return 0;
decoder->pos -= part_psb; decoder->next_buf = decoder->buf + rest_psb; decoder->next_len = decoder->len - rest_psb; memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); decoder->buf = decoder->temp_buf; decoder->len = INTEL_PT_PSB_LEN;
return 0;}
static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder){ unsigned char *next; int ret;
intel_pt_log("Scanning for PSB\n"); while (1) { if (!decoder->len) { ret = intel_pt_get_next_data(decoder, false); if (ret) return ret; }
next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); if (!next) { int part_psb;
part_psb = intel_pt_part_psb(decoder); if (part_psb) { ret = intel_pt_get_split_psb(decoder, part_psb); if (ret) return ret; } else { decoder->pos += decoder->len; decoder->len = 0; } continue; }
decoder->pkt_step = next - decoder->buf; return intel_pt_get_next_packet(decoder); }}
static int intel_pt_sync(struct intel_pt_decoder *decoder){ int err;
decoder->pge = false; decoder->continuous_period = false; decoder->have_last_ip = false; decoder->last_ip = 0; decoder->psb_ip = 0; decoder->ip = 0; intel_pt_clear_stack(&decoder->stack);
err = intel_pt_scan_for_psb(decoder); if (err) return err;
if (decoder->vm_time_correlation) { decoder->in_psb = true; if (!decoder->timestamp) decoder->timestamp = 1; decoder->state.type = 0; decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; return 0; }
decoder->have_last_ip = true; decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
err = intel_pt_walk_psb(decoder); if (err) return err;
decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ decoder->state.from_ip = decoder->psb_ip; decoder->state.to_ip = 0;
if (decoder->ip) { /* * In hop mode, resample to get the PSB FUP ip as an * "instruction" sample. */ if (decoder->hop) decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; else decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; }
return 0;}
static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder){ uint64_t est = decoder->sample_insn_cnt << 1;
if (!decoder->cbr || !decoder->max_non_turbo_ratio) goto out;
est *= decoder->max_non_turbo_ratio; est /= decoder->cbr;out: return decoder->sample_timestamp + est;}
const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder){ int err;
do { decoder->state.type = INTEL_PT_BRANCH; decoder->state.flags = 0;
switch (decoder->pkt_state) { case INTEL_PT_STATE_NO_PSB: err = intel_pt_sync(decoder); break; case INTEL_PT_STATE_NO_IP: decoder->have_last_ip = false; decoder->last_ip = 0; decoder->ip = 0; fallthrough; case INTEL_PT_STATE_ERR_RESYNC: err = intel_pt_sync_ip(decoder); break; case INTEL_PT_STATE_IN_SYNC: err = intel_pt_walk_trace(decoder); break; case INTEL_PT_STATE_TNT: case INTEL_PT_STATE_TNT_CONT: err = intel_pt_walk_tnt(decoder); if (err == -EAGAIN) err = intel_pt_walk_trace(decoder); break; case INTEL_PT_STATE_TIP: case INTEL_PT_STATE_TIP_PGD: err = intel_pt_walk_tip(decoder); break; case INTEL_PT_STATE_FUP: err = intel_pt_walk_fup(decoder); if (err == -EAGAIN) err = intel_pt_walk_fup_tip(decoder); break; case INTEL_PT_STATE_FUP_NO_TIP: err = intel_pt_walk_fup(decoder); if (err == -EAGAIN) err = intel_pt_walk_trace(decoder); break; case INTEL_PT_STATE_FUP_IN_PSB: err = intel_pt_fup_in_psb(decoder); break; case INTEL_PT_STATE_RESAMPLE: err = intel_pt_resample(decoder); break; case INTEL_PT_STATE_VM_TIME_CORRELATION: err = intel_pt_vm_time_correlation(decoder); break; default: err = intel_pt_bug(decoder); break; } } while (err == -ENOLINK);
if (err) { decoder->state.err = intel_pt_ext_err(err); if (err != -EOVERFLOW) decoder->state.from_ip = decoder->ip; intel_pt_update_sample_time(decoder); decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; intel_pt_set_nr(decoder); } else { decoder->state.err = 0; if (decoder->cbr != decoder->cbr_seen) { decoder->cbr_seen = decoder->cbr; if (!decoder->state.type) { decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; } decoder->state.type |= INTEL_PT_CBR_CHG; decoder->state.cbr_payload = decoder->cbr_payload; decoder->state.cbr = decoder->cbr; } if (intel_pt_sample_time(decoder->pkt_state)) { intel_pt_update_sample_time(decoder); if (decoder->sample_cyc) { decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; decoder->state.flags |= INTEL_PT_SAMPLE_IPC; decoder->sample_cyc = false; } } /* * When using only TSC/MTC to compute cycles, IPC can be * sampled as soon as the cycle count changes. */ if (!decoder->have_cyc) decoder->state.flags |= INTEL_PT_SAMPLE_IPC; }
/* Let PSB event always have TSC timestamp */ if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) decoder->sample_timestamp = decoder->tsc_timestamp;
decoder->state.from_nr = decoder->nr; decoder->state.to_nr = decoder->next_nr; decoder->nr = decoder->next_nr;
decoder->state.timestamp = decoder->sample_timestamp; decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
return &decoder->state;}
/** * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. * @buf: pointer to buffer pointer * @len: size of buffer * * Updates the buffer pointer to point to the start of the next PSB packet if * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, * @len is adjusted accordingly. * * Return: %true if a PSB packet is found, %false otherwise. */static bool intel_pt_next_psb(unsigned char **buf, size_t *len){ unsigned char *next;
next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); if (next) { *len -= next - *buf; *buf = next; return true; } return false;}
/** * intel_pt_step_psb - move buffer pointer to the start of the following PSB * packet. * @buf: pointer to buffer pointer * @len: size of buffer * * Updates the buffer pointer to point to the start of the following PSB packet * (skipping the PSB at @buf itself) if there is one, otherwise the buffer * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. * * Return: %true if a PSB packet is found, %false otherwise. */static bool intel_pt_step_psb(unsigned char **buf, size_t *len){ unsigned char *next;
if (!*len) return false;
next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); if (next) { *len -= next - *buf; *buf = next; return true; } return false;}
/** * intel_pt_last_psb - find the last PSB packet in a buffer. * @buf: buffer * @len: size of buffer * * This function finds the last PSB in a buffer. * * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. */static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len){ const char *n = INTEL_PT_PSB_STR; unsigned char *p; size_t k;
if (len < INTEL_PT_PSB_LEN) return NULL;
k = len - INTEL_PT_PSB_LEN + 1; while (1) { p = memrchr(buf, n[0], k); if (!p) return NULL; if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) return p; k = p - buf; if (!k) return NULL; }}
/** * intel_pt_next_tsc - find and return next TSC. * @buf: buffer * @len: size of buffer * @tsc: TSC value returned * @rem: returns remaining size when TSC is found * * Find a TSC packet in @buf and return the TSC value. This function assumes * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a * PSBEND packet is found. * * Return: %true if TSC is found, false otherwise. */static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, size_t *rem){ enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; struct intel_pt_pkt packet; int ret;
while (len) { ret = intel_pt_get_packet(buf, len, &packet, &ctx); if (ret <= 0) return false; if (packet.type == INTEL_PT_TSC) { *tsc = packet.payload; *rem = len; return true; } if (packet.type == INTEL_PT_PSBEND) return false; buf += ret; len -= ret; } return false;}
/** * intel_pt_tsc_cmp - compare 7-byte TSCs. * @tsc1: first TSC to compare * @tsc2: second TSC to compare * * This function compares 7-byte TSC values allowing for the possibility that * TSC wrapped around. Generally it is not possible to know if TSC has wrapped * around so for that purpose this function assumes the absolute difference is * less than half the maximum difference. * * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is * after @tsc2. */static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2){ const uint64_t halfway = (1ULL << 55);
if (tsc1 == tsc2) return 0;
if (tsc1 < tsc2) { if (tsc2 - tsc1 < halfway) return -1; else return 1; } else { if (tsc1 - tsc2 < halfway) return 1; else return -1; }}
#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
/** * adj_for_padding - adjust overlap to account for padding. * @buf_b: second buffer * @buf_a: first buffer * @len_a: size of first buffer * * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap * accordingly. * * Return: A pointer into @buf_b from where non-overlapped data starts */static unsigned char *adj_for_padding(unsigned char *buf_b, unsigned char *buf_a, size_t len_a){ unsigned char *p = buf_b - MAX_PADDING; unsigned char *q = buf_a + len_a - MAX_PADDING; int i;
for (i = MAX_PADDING; i; i--, p++, q++) { if (*p != *q) break; }
return p;}
/** * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data * using TSC. * @buf_a: first buffer * @len_a: size of first buffer * @buf_b: second buffer * @len_b: size of second buffer * @consecutive: returns true if there is data in buf_b that is consecutive * to buf_a * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling * * If the trace contains TSC we can look at the last TSC of @buf_a and the * first TSC of @buf_b in order to determine if the buffers overlap, and then * walk forward in @buf_b until a later TSC is found. A precondition is that * @buf_a and @buf_b are positioned at a PSB. * * Return: A pointer into @buf_b from where non-overlapped data starts, or * @buf_b + @len_b if there is no non-overlapped data. */static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, bool *consecutive, bool ooo_tsc){ uint64_t tsc_a, tsc_b; unsigned char *p; size_t len, rem_a, rem_b;
p = intel_pt_last_psb(buf_a, len_a); if (!p) return buf_b; /* No PSB in buf_a => no overlap */
len = len_a - (p - buf_a); if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { /* The last PSB+ in buf_a is incomplete, so go back one more */ len_a -= len; p = intel_pt_last_psb(buf_a, len_a); if (!p) return buf_b; /* No full PSB+ => assume no overlap */ len = len_a - (p - buf_a); if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) return buf_b; /* No TSC in buf_a => assume no overlap */ }
while (1) { /* Ignore PSB+ with no TSC */ if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
/* Same TSC, so buffers are consecutive */ if (!cmp && rem_b >= rem_a) { unsigned char *start;
*consecutive = true; start = buf_b + len_b - (rem_b - rem_a); return adj_for_padding(start, buf_a, len_a); } if (cmp < 0 && !ooo_tsc) return buf_b; /* tsc_a < tsc_b => no overlap */ }
if (!intel_pt_step_psb(&buf_b, &len_b)) return buf_b + len_b; /* No PSB in buf_b => no data */ }}
/** * intel_pt_find_overlap - determine start of non-overlapped trace data. * @buf_a: first buffer * @len_a: size of first buffer * @buf_b: second buffer * @len_b: size of second buffer * @have_tsc: can use TSC packets to detect overlap * @consecutive: returns true if there is data in buf_b that is consecutive * to buf_a * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling * * When trace samples or snapshots are recorded there is the possibility that * the data overlaps. Note that, for the purposes of decoding, data is only * useful if it begins with a PSB packet. * * Return: A pointer into @buf_b from where non-overlapped data starts, or * @buf_b + @len_b if there is no non-overlapped data. */unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, bool have_tsc, bool *consecutive, bool ooo_tsc){ unsigned char *found;
/* Buffer 'b' must start at PSB so throw away everything before that */ if (!intel_pt_next_psb(&buf_b, &len_b)) return buf_b + len_b; /* No PSB */
if (!intel_pt_next_psb(&buf_a, &len_a)) return buf_b; /* No overlap */
if (have_tsc) { found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, consecutive, ooo_tsc); if (found) return found; }
/* * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, * we can ignore the first part of buffer 'a'. */ while (len_b < len_a) { if (!intel_pt_step_psb(&buf_a, &len_a)) return buf_b; /* No overlap */ }
/* Now len_b >= len_a */ while (1) { /* Potential overlap so check the bytes */ found = memmem(buf_a, len_a, buf_b, len_a); if (found) { *consecutive = true; return adj_for_padding(buf_b + len_a, buf_a, len_a); }
/* Try again at next PSB in buffer 'a' */ if (!intel_pt_step_psb(&buf_a, &len_a)) return buf_b; /* No overlap */ }}
/** * struct fast_forward_data - data used by intel_pt_ff_cb(). * @timestamp: timestamp to fast forward towards * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than * the fast forward timestamp. */struct fast_forward_data { uint64_t timestamp; uint64_t buf_timestamp;};
/** * intel_pt_ff_cb - fast forward lookahead callback. * @buffer: Intel PT trace buffer * @data: opaque pointer to fast forward data (struct fast_forward_data) * * Determine if @buffer trace is past the fast forward timestamp. * * Return: 1 (stop lookahead) if @buffer trace is past the fast forward * timestamp, and 0 otherwise. */static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data){ struct fast_forward_data *d = data; unsigned char *buf; uint64_t tsc; size_t rem; size_t len;
buf = (unsigned char *)buffer->buf; len = buffer->len;
if (!intel_pt_next_psb(&buf, &len) || !intel_pt_next_tsc(buf, len, &tsc, &rem)) return 0;
tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", tsc, buffer->ref_timestamp);
/* * If the buffer contains a timestamp earlier that the fast forward * timestamp, then record it, else stop. */ if (tsc < d->timestamp) d->buf_timestamp = buffer->ref_timestamp; else return 1;
return 0;}
/** * intel_pt_fast_forward - reposition decoder forwards. * @decoder: Intel PT decoder * @timestamp: timestamp to fast forward towards * * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. * * Return: 0 on success or negative error code on failure. */int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp){ struct fast_forward_data d = { .timestamp = timestamp }; unsigned char *buf; size_t len; int err;
intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
/* Find buffer timestamp of buffer to fast forward to */ err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); if (err < 0) return err;
/* Walk to buffer with same buffer timestamp */ if (d.buf_timestamp) { do { decoder->pos += decoder->len; decoder->len = 0; err = intel_pt_get_next_data(decoder, true); /* -ENOLINK means non-consecutive trace */ if (err && err != -ENOLINK) return err; } while (decoder->buf_timestamp != d.buf_timestamp); }
if (!decoder->buf) return 0;
buf = (unsigned char *)decoder->buf; len = decoder->len;
if (!intel_pt_next_psb(&buf, &len)) return 0;
/* * Walk PSBs while the PSB timestamp is less than the fast forward * timestamp. */ do { uint64_t tsc; size_t rem;
if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) break; tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); /* * A TSC packet can slip past MTC packets but, after fast * forward, decoding starts at the TSC timestamp. That means * the timestamps may not be exactly the same as the timestamps * that would have been decoded without fast forward. */ if (tsc < timestamp) { intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); decoder->pos += decoder->len - len; decoder->buf = buf; decoder->len = len; intel_pt_reposition(decoder); } else { break; } } while (intel_pt_step_psb(&buf, &len));
return 0;}