Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590135911359213593135941359513596135971359813599136001360113602136031360413605136061360713608136091361013611136121361313614136151361613617136181361913620136211362213623136241362513626136271362813629136301363113632136331363413635136361363713638136391364013641136421364313644136451364613647136481364913650136511365213653136541365513656136571365813659136601366113662136631366413665136661366713668136691367013671136721367313674136751367613677136781367913680136811368213683136841368513686136871368813689136901369113692136931369413695136961369713698136991370013701137021370313704137051370613707137081370913710137111371213713137141371513716137171371813719137201372113722137231372413725137261372713728137291373013731137321373313734137351373613737137381373913740137411374213743137441374513746137471374813749137501375113752137531375413755137561375713758137591376013761137621376313764137651376613767137681376913770137711377213773137741377513776137771377813779137801378113782137831378413785137861378713788137891379013791137921379313794137951379613797137981379913800138011380213803138041380513806138071380813809138101381113812138131381413815138161381713818138191382013821138221382313824138251382613827138281382913830138311383213833138341383513836138371383813839138401384113842138431384413845138461384713848138491385013851138521385313854138551385613857138581385913860138611386213863138641386513866138671386813869138701387113872138731387413875138761387713878138791388013881138821388313884138851388613887138881388913890138911389213893138941389513896138971389813899139001390113902139031390413905139061390713908139091391013911139121391313914139151391613917139181391913920139211392213923139241392513926139271392813929139301393113932139331393413935139361393713938139391394013941139421394313944139451394613947139481394913950139511395213953139541395513956139571395813959139601396113962139631396413965139661396713968139691397013971139721397313974139751397613977139781397913980139811398213983139841398513986139871398813989139901399113992139931399413995139961399713998139991400014001140021400314004140051400614007140081400914010140111401214013140141401514016140171401814019140201402114022140231402414025140261402714028140291403014031140321403314034140351403614037140381403914040140411404214043140441404514046140471404814049140501405114052140531405414055140561405714058140591406014061140621406314064140651406614067140681406914070140711407214073140741407514076140771407814079140801408114082140831408414085140861408714088140891409014091140921409314094140951409614097140981409914100141011410214103141041410514106141071410814109141101411114112141131411414115141161411714118141191412014121141221412314124141251412614127141281412914130141311413214133141341413514136141371413814139141401414114142141431414414145141461414714148141491415014151141521415314154141551415614157141581415914160141611416214163141641416514166141671416814169141701417114172141731417414175141761417714178141791418014181141821418314184141851418614187141881418914190141911419214193141941419514196141971419814199142001420114202142031420414205142061420714208142091421014211142121421314214142151421614217142181421914220142211422214223142241422514226142271422814229142301423114232142331423414235142361423714238142391424014241142421424314244142451424614247142481424914250142511425214253142541425514256142571425814259142601426114262142631426414265142661426714268142691427014271142721427314274142751427614277142781427914280142811428214283142841428514286142871428814289142901429114292142931429414295142961429714298142991430014301143021430314304143051430614307143081430914310143111431214313143141431514316143171431814319143201432114322143231432414325143261432714328143291433014331143321433314334143351433614337143381433914340143411434214343143441434514346143471434814349143501435114352143531435414355143561435714358143591436014361143621436314364143651436614367143681436914370143711437214373143741437514376143771437814379143801438114382143831438414385143861438714388143891439014391143921439314394143951439614397143981439914400144011440214403144041440514406144071440814409144101441114412144131441414415144161441714418144191442014421144221442314424144251442614427144281442914430144311443214433144341443514436144371443814439144401444114442144431444414445144461444714448144491445014451144521445314454144551445614457144581445914460144611446214463144641446514466144671446814469144701447114472144731447414475144761447714478144791448014481144821448314484144851448614487144881448914490144911449214493144941449514496144971449814499145001450114502145031450414505145061450714508145091451014511145121451314514145151451614517145181451914520145211452214523145241452514526145271452814529145301453114532145331453414535145361453714538145391454014541145421454314544145451454614547145481454914550145511455214553145541455514556145571455814559145601456114562145631456414565145661456714568145691457014571145721457314574145751457614577145781457914580145811458214583145841458514586145871458814589145901459114592145931459414595145961459714598145991460014601146021460314604146051460614607146081460914610146111461214613146141461514616146171461814619146201462114622146231462414625146261462714628146291463014631146321463314634146351463614637146381463914640146411464214643146441464514646146471464814649146501465114652146531465414655146561465714658146591466014661146621466314664146651466614667146681466914670146711467214673146741467514676146771467814679146801468114682146831468414685146861468714688146891469014691146921469314694146951469614697146981469914700147011470214703147041470514706147071470814709147101471114712147131471414715147161471714718147191472014721147221472314724147251472614727147281472914730147311473214733147341473514736147371473814739147401474114742147431474414745147461474714748147491475014751147521475314754147551475614757147581475914760147611476214763147641476514766147671476814769147701477114772147731477414775147761477714778147791478014781147821478314784147851478614787147881478914790147911479214793147941479514796147971479814799148001480114802148031480414805148061480714808148091481014811148121481314814148151481614817148181481914820148211482214823148241482514826148271482814829148301483114832148331483414835148361483714838148391484014841148421484314844148451484614847148481484914850148511485214853148541485514856148571485814859148601486114862148631486414865148661486714868148691487014871148721487314874148751487614877148781487914880148811488214883148841488514886148871488814889148901489114892148931489414895148961489714898148991490014901149021490314904149051490614907149081490914910149111491214913149141491514916149171491814919149201492114922149231492414925149261492714928149291493014931149321493314934149351493614937149381493914940149411494214943149441494514946149471494814949149501495114952149531495414955149561495714958149591496014961149621496314964149651496614967149681496914970149711497214973149741497514976149771497814979149801498114982149831498414985149861498714988149891499014991149921499314994149951499614997149981499915000150011500215003150041500515006150071500815009150101501115012150131501415015150161501715018150191502015021150221502315024150251502615027150281502915030150311503215033150341503515036150371503815039150401504115042150431504415045150461504715048150491505015051150521505315054150551505615057150581505915060150611506215063150641506515066150671506815069150701507115072150731507415075150761507715078150791508015081150821508315084150851508615087150881508915090150911509215093150941509515096150971509815099151001510115102151031510415105151061510715108151091511015111151121511315114151151511615117151181511915120151211512215123151241512515126151271512815129151301513115132151331513415135151361513715138151391514015141151421514315144151451514615147151481514915150151511515215153151541515515156151571515815159151601516115162151631516415165151661516715168151691517015171151721517315174151751517615177151781517915180151811518215183151841518515186151871518815189151901519115192151931519415195151961519715198151991520015201152021520315204152051520615207152081520915210152111521215213152141521515216152171521815219152201522115222152231522415225152261522715228152291523015231152321523315234152351523615237152381523915240152411524215243152441524515246152471524815249152501525115252152531525415255152561525715258152591526015261152621526315264152651526615267152681526915270152711527215273152741527515276152771527815279152801528115282152831528415285152861528715288152891529015291152921529315294152951529615297152981529915300153011530215303153041530515306153071530815309153101531115312153131531415315153161531715318153191532015321153221532315324153251532615327153281532915330153311533215333153341533515336153371533815339153401534115342153431534415345153461534715348153491535015351153521535315354153551535615357153581535915360153611536215363153641536515366153671536815369153701537115372153731537415375153761537715378153791538015381153821538315384153851538615387153881538915390153911539215393153941539515396153971539815399154001540115402154031540415405154061540715408154091541015411154121541315414154151541615417154181541915420154211542215423154241542515426154271542815429154301543115432154331543415435154361543715438154391544015441154421544315444154451544615447154481544915450154511545215453154541545515456154571545815459154601546115462154631546415465154661546715468154691547015471154721547315474154751547615477154781547915480154811548215483154841548515486154871548815489154901549115492154931549415495154961549715498154991550015501155021550315504155051550615507155081550915510155111551215513155141551515516155171551815519155201552115522155231552415525155261552715528155291553015531155321553315534155351553615537155381553915540155411554215543155441554515546155471554815549155501555115552155531555415555155561555715558155591556015561155621556315564155651556615567155681556915570155711557215573155741557515576155771557815579155801558115582155831558415585155861558715588155891559015591155921559315594155951559615597155981559915600156011560215603156041560515606156071560815609156101561115612156131561415615156161561715618156191562015621156221562315624156251562615627156281562915630156311563215633156341563515636156371563815639156401564115642156431564415645156461564715648156491565015651156521565315654156551565615657156581565915660156611566215663156641566515666156671566815669156701567115672156731567415675156761567715678156791568015681156821568315684156851568615687156881568915690156911569215693156941569515696156971569815699157001570115702157031570415705157061570715708157091571015711157121571315714157151571615717157181571915720157211572215723157241572515726157271572815729157301573115732157331573415735157361573715738157391574015741157421574315744157451574615747157481574915750157511575215753157541575515756157571575815759157601576115762157631576415765157661576715768157691577015771157721577315774157751577615777157781577915780157811578215783157841578515786157871578815789157901579115792157931579415795157961579715798157991580015801158021580315804158051580615807158081580915810158111581215813158141581515816158171581815819158201582115822158231582415825158261582715828158291583015831158321583315834158351583615837158381583915840158411584215843158441584515846158471584815849158501585115852158531585415855158561585715858158591586015861158621586315864158651586615867158681586915870158711587215873158741587515876158771587815879158801588115882158831588415885158861588715888158891589015891158921589315894158951589615897158981589915900159011590215903159041590515906159071590815909159101591115912159131591415915159161591715918159191592015921159221592315924159251592615927159281592915930159311593215933159341593515936159371593815939159401594115942159431594415945159461594715948159491595015951159521595315954159551595615957159581595915960159611596215963159641596515966159671596815969159701597115972159731597415975159761597715978159791598015981159821598315984159851598615987159881598915990159911599215993159941599515996159971599815999160001600116002160031600416005160061600716008160091601016011160121601316014160151601616017160181601916020160211602216023160241602516026160271602816029160301603116032160331603416035160361603716038160391604016041160421604316044160451604616047160481604916050160511605216053160541605516056160571605816059160601606116062160631606416065160661606716068160691607016071160721607316074160751607616077160781607916080160811608216083160841608516086160871608816089160901609116092160931609416095160961609716098160991610016101161021610316104161051610616107161081610916110161111611216113161141611516116161171611816119161201612116122161231612416125161261612716128161291613016131161321613316134161351613616137161381613916140161411614216143161441614516146161471614816149161501615116152161531615416155161561615716158161591616016161161621616316164161651616616167161681616916170161711617216173161741617516176161771617816179161801618116182161831618416185161861618716188161891619016191161921619316194161951619616197161981619916200162011620216203162041620516206162071620816209162101621116212162131621416215162161621716218162191622016221162221622316224162251622616227162281622916230162311623216233162341623516236162371623816239162401624116242162431624416245162461624716248162491625016251162521625316254162551625616257162581625916260162611626216263162641626516266162671626816269162701627116272162731627416275162761627716278162791628016281162821628316284162851628616287162881628916290162911629216293162941629516296162971629816299163001630116302163031630416305163061630716308163091631016311163121631316314163151631616317163181631916320163211632216323163241632516326163271632816329163301633116332163331633416335163361633716338163391634016341163421634316344163451634616347163481634916350163511635216353163541635516356163571635816359163601636116362163631636416365163661636716368163691637016371163721637316374163751637616377163781637916380163811638216383163841638516386163871638816389163901639116392163931639416395163961639716398163991640016401164021640316404164051640616407164081640916410164111641216413164141641516416164171641816419164201642116422164231642416425164261642716428164291643016431164321643316434164351643616437164381643916440164411644216443164441644516446164471644816449164501645116452164531645416455164561645716458164591646016461164621646316464164651646616467164681646916470164711647216473164741647516476164771647816479164801648116482164831648416485164861648716488164891649016491164921649316494164951649616497164981649916500165011650216503165041650516506165071650816509165101651116512165131651416515165161651716518165191652016521165221652316524165251652616527165281652916530165311653216533165341653516536165371653816539165401654116542165431654416545165461654716548165491655016551165521655316554165551655616557165581655916560165611656216563165641656516566165671656816569165701657116572165731657416575165761657716578165791658016581165821658316584165851658616587165881658916590165911659216593165941659516596165971659816599166001660116602166031660416605166061660716608166091661016611166121661316614166151661616617166181661916620166211662216623166241662516626166271662816629166301663116632166331663416635166361663716638166391664016641166421664316644166451664616647166481664916650166511665216653166541665516656166571665816659166601666116662166631666416665166661666716668166691667016671166721667316674166751667616677166781667916680166811668216683166841668516686166871668816689166901669116692166931669416695166961669716698166991670016701167021670316704167051670616707167081670916710167111671216713167141671516716167171671816719167201672116722167231672416725167261672716728167291673016731167321673316734167351673616737167381673916740167411674216743167441674516746167471674816749167501675116752167531675416755167561675716758167591676016761167621676316764167651676616767167681676916770167711677216773167741677516776167771677816779167801678116782167831678416785167861678716788167891679016791167921679316794167951679616797167981679916800168011680216803168041680516806168071680816809168101681116812168131681416815168161681716818168191682016821168221682316824168251682616827168281682916830168311683216833168341683516836168371683816839168401684116842168431684416845168461684716848168491685016851168521685316854168551685616857168581685916860168611686216863168641686516866168671686816869168701687116872168731687416875168761687716878168791688016881168821688316884168851688616887168881688916890168911689216893168941689516896168971689816899169001690116902169031690416905169061690716908169091691016911169121691316914169151691616917169181691916920169211692216923169241692516926169271692816929169301693116932169331693416935169361693716938169391694016941169421694316944169451694616947169481694916950169511695216953169541695516956169571695816959169601696116962169631696416965169661696716968169691697016971169721697316974169751697616977169781697916980169811698216983169841698516986169871698816989169901699116992169931699416995169961699716998169991700017001170021700317004170051700617007170081700917010170111701217013170141701517016170171701817019170201702117022170231702417025170261702717028170291703017031170321703317034170351703617037170381703917040170411704217043170441704517046170471704817049170501705117052170531705417055170561705717058170591706017061170621706317064170651706617067170681706917070170711707217073170741707517076170771707817079170801708117082170831708417085170861708717088170891709017091170921709317094170951709617097170981709917100171011710217103171041710517106171071710817109171101711117112171131711417115171161711717118171191712017121171221712317124171251712617127171281712917130171311713217133171341713517136171371713817139171401714117142171431714417145171461714717148171491715017151171521715317154171551715617157171581715917160171611716217163171641716517166171671716817169171701717117172171731717417175171761717717178171791718017181171821718317184171851718617187171881718917190171911719217193171941719517196171971719817199172001720117202172031720417205172061720717208172091721017211172121721317214172151721617217172181721917220172211722217223172241722517226172271722817229172301723117232172331723417235172361723717238172391724017241172421724317244172451724617247172481724917250172511725217253172541725517256172571725817259172601726117262172631726417265172661726717268172691727017271172721727317274172751727617277172781727917280172811728217283172841728517286172871728817289172901729117292172931729417295172961729717298172991730017301173021730317304173051730617307173081730917310173111731217313173141731517316173171731817319173201732117322173231732417325173261732717328173291733017331173321733317334173351733617337173381733917340173411734217343173441734517346173471734817349173501735117352173531735417355173561735717358173591736017361173621736317364173651736617367173681736917370173711737217373173741737517376173771737817379173801738117382173831738417385173861738717388173891739017391173921739317394173951739617397173981739917400174011740217403174041740517406174071740817409174101741117412174131741417415174161741717418174191742017421174221742317424174251742617427174281742917430174311743217433174341743517436174371743817439174401744117442174431744417445174461744717448174491745017451174521745317454174551745617457174581745917460174611746217463174641746517466174671746817469174701747117472174731747417475174761747717478174791748017481174821748317484174851748617487174881748917490174911749217493174941749517496174971749817499175001750117502175031750417505175061750717508175091751017511175121751317514175151751617517175181751917520175211752217523175241752517526175271752817529175301753117532175331753417535175361753717538175391754017541175421754317544175451754617547175481754917550175511755217553175541755517556175571755817559175601756117562175631756417565175661756717568175691757017571175721757317574175751757617577175781757917580175811758217583175841758517586175871758817589175901759117592175931759417595175961759717598175991760017601176021760317604176051760617607176081760917610176111761217613176141761517616176171761817619176201762117622176231762417625176261762717628176291763017631176321763317634176351763617637176381763917640176411764217643176441764517646176471764817649176501765117652176531765417655176561765717658176591766017661176621766317664176651766617667176681766917670176711767217673176741767517676176771767817679176801768117682176831768417685176861768717688176891769017691176921769317694176951769617697176981769917700177011770217703177041770517706177071770817709177101771117712177131771417715177161771717718177191772017721177221772317724177251772617727177281772917730177311773217733177341773517736177371773817739177401774117742177431774417745177461774717748177491775017751177521775317754177551775617757177581775917760177611776217763177641776517766177671776817769177701777117772177731777417775177761777717778177791778017781177821778317784177851778617787177881778917790177911779217793177941779517796177971779817799178001780117802178031780417805178061780717808178091781017811178121781317814178151781617817178181781917820178211782217823178241782517826178271782817829178301783117832178331783417835178361783717838178391784017841178421784317844178451784617847178481784917850178511785217853178541785517856178571785817859178601786117862178631786417865178661786717868178691787017871178721787317874178751787617877178781787917880178811788217883178841788517886178871788817889178901789117892178931789417895178961789717898178991790017901179021790317904179051790617907179081790917910179111791217913179141791517916179171791817919179201792117922179231792417925179261792717928179291793017931179321793317934179351793617937179381793917940179411794217943179441794517946179471794817949179501795117952179531795417955179561795717958179591796017961179621796317964179651796617967179681796917970179711797217973179741797517976179771797817979179801798117982179831798417985179861798717988179891799017991179921799317994179951799617997179981799918000180011800218003180041800518006180071800818009180101801118012180131801418015180161801718018180191802018021180221802318024180251802618027180281802918030180311803218033180341803518036180371803818039180401804118042180431804418045180461804718048180491805018051180521805318054180551805618057180581805918060180611806218063180641806518066180671806818069180701807118072180731807418075180761807718078180791808018081180821808318084180851808618087180881808918090180911809218093180941809518096180971809818099181001810118102181031810418105181061810718108181091811018111181121811318114181151811618117181181811918120181211812218123181241812518126181271812818129181301813118132181331813418135181361813718138181391814018141181421814318144181451814618147181481814918150181511815218153181541815518156181571815818159181601816118162181631816418165181661816718168181691817018171181721817318174181751817618177181781817918180181811818218183181841818518186181871818818189181901819118192181931819418195181961819718198181991820018201182021820318204182051820618207182081820918210182111821218213182141821518216182171821818219182201822118222182231822418225182261822718228182291823018231182321823318234182351823618237182381823918240182411824218243182441824518246182471824818249182501825118252182531825418255182561825718258182591826018261182621826318264182651826618267182681826918270182711827218273182741827518276182771827818279182801828118282182831828418285182861828718288182891829018291182921829318294182951829618297182981829918300183011830218303183041830518306183071830818309183101831118312183131831418315183161831718318183191832018321183221832318324183251832618327183281832918330183311833218333183341833518336183371833818339183401834118342183431834418345183461834718348183491835018351183521835318354183551835618357183581835918360183611836218363183641836518366183671836818369183701837118372183731837418375183761837718378183791838018381183821838318384183851838618387183881838918390183911839218393183941839518396183971839818399184001840118402184031840418405184061840718408184091841018411184121841318414184151841618417184181841918420184211842218423184241842518426184271842818429184301843118432184331843418435184361843718438184391844018441184421844318444184451844618447184481844918450184511845218453184541845518456184571845818459184601846118462184631846418465184661846718468184691847018471184721847318474184751847618477184781847918480184811848218483184841848518486184871848818489184901849118492184931849418495184961849718498184991850018501185021850318504185051850618507185081850918510185111851218513185141851518516185171851818519185201852118522185231852418525185261852718528185291853018531185321853318534185351853618537185381853918540185411854218543185441854518546185471854818549185501855118552185531855418555185561855718558185591856018561185621856318564185651856618567185681856918570185711857218573185741857518576185771857818579185801858118582185831858418585185861858718588185891859018591185921859318594185951859618597185981859918600186011860218603186041860518606186071860818609186101861118612186131861418615186161861718618186191862018621186221862318624186251862618627186281862918630186311863218633186341863518636186371863818639186401864118642186431864418645186461864718648186491865018651186521865318654186551865618657186581865918660186611866218663186641866518666186671866818669186701867118672186731867418675186761867718678186791868018681186821868318684186851868618687186881868918690186911869218693186941869518696186971869818699187001870118702187031870418705187061870718708187091871018711187121871318714187151871618717187181871918720187211872218723187241872518726187271872818729187301873118732187331873418735187361873718738187391874018741187421874318744187451874618747187481874918750187511875218753187541875518756187571875818759187601876118762187631876418765187661876718768187691877018771187721877318774187751877618777187781877918780187811878218783187841878518786187871878818789187901879118792187931879418795187961879718798187991880018801188021880318804188051880618807188081880918810188111881218813188141881518816188171881818819188201882118822188231882418825188261882718828188291883018831188321883318834188351883618837188381883918840188411884218843188441884518846188471884818849188501885118852188531885418855188561885718858188591886018861188621886318864188651886618867188681886918870188711887218873188741887518876188771887818879188801888118882188831888418885188861888718888188891889018891188921889318894188951889618897188981889918900189011890218903189041890518906189071890818909189101891118912189131891418915189161891718918189191892018921189221892318924189251892618927189281892918930189311893218933189341893518936189371893818939189401894118942189431894418945189461894718948189491895018951189521895318954189551895618957189581895918960189611896218963189641896518966189671896818969189701897118972189731897418975189761897718978189791898018981189821898318984189851898618987189881898918990189911899218993189941899518996189971899818999190001900119002190031900419005190061900719008190091901019011190121901319014190151901619017190181901919020190211902219023190241902519026190271902819029190301903119032190331903419035190361903719038190391904019041190421904319044190451904619047190481904919050190511905219053190541905519056190571905819059190601906119062190631906419065190661906719068190691907019071190721907319074190751907619077190781907919080190811908219083190841908519086190871908819089190901909119092190931909419095190961909719098190991910019101191021910319104191051910619107191081910919110191111911219113191141911519116191171911819119191201912119122191231912419125191261912719128191291913019131191321913319134191351913619137191381913919140191411914219143191441914519146191471914819149191501915119152191531915419155191561915719158191591916019161191621916319164191651916619167191681916919170191711917219173191741917519176191771917819179191801918119182191831918419185191861918719188191891919019191191921919319194191951919619197191981919919200192011920219203192041920519206192071920819209192101921119212192131921419215192161921719218192191922019221192221922319224192251922619227192281922919230192311923219233192341923519236192371923819239192401924119242192431924419245192461924719248192491925019251192521925319254192551925619257192581925919260192611926219263192641926519266192671926819269192701927119272192731927419275192761927719278192791928019281192821928319284192851928619287192881928919290192911929219293192941929519296192971929819299193001930119302193031930419305193061930719308193091931019311193121931319314193151931619317193181931919320193211932219323193241932519326193271932819329193301933119332193331933419335193361933719338193391934019341193421934319344193451934619347193481934919350193511935219353193541935519356193571935819359193601936119362193631936419365193661936719368193691937019371193721937319374193751937619377193781937919380193811938219383193841938519386193871938819389193901939119392193931939419395193961939719398193991940019401194021940319404194051940619407194081940919410194111941219413194141941519416194171941819419194201942119422194231942419425194261942719428194291943019431194321943319434194351943619437194381943919440194411944219443194441944519446194471944819449194501945119452194531945419455194561945719458194591946019461194621946319464194651946619467194681946919470194711947219473194741947519476194771947819479194801948119482194831948419485194861948719488194891949019491194921949319494194951949619497194981949919500195011950219503195041950519506195071950819509195101951119512195131951419515195161951719518195191952019521195221952319524195251952619527195281952919530195311953219533195341953519536195371953819539195401954119542195431954419545195461954719548195491955019551195521955319554195551955619557195581955919560195611956219563195641956519566195671956819569195701957119572195731957419575195761957719578195791958019581195821958319584195851958619587195881958919590195911959219593195941959519596195971959819599196001960119602196031960419605196061960719608196091961019611196121961319614196151961619617196181961919620196211962219623196241962519626196271962819629196301963119632196331963419635196361963719638196391964019641196421964319644196451964619647196481964919650196511965219653196541965519656196571965819659196601966119662196631966419665196661966719668196691967019671196721967319674196751967619677196781967919680196811968219683196841968519686196871968819689196901969119692196931969419695196961969719698196991970019701197021970319704197051970619707197081970919710197111971219713197141971519716197171971819719197201972119722197231972419725197261972719728197291973019731197321973319734197351973619737197381973919740197411974219743197441974519746197471974819749197501975119752197531975419755197561975719758197591976019761197621976319764197651976619767197681976919770197711977219773197741977519776197771977819779197801978119782197831978419785197861978719788197891979019791197921979319794197951979619797197981979919800198011980219803198041980519806198071980819809198101981119812198131981419815198161981719818198191982019821198221982319824198251982619827198281982919830198311983219833198341983519836198371983819839198401984119842198431984419845198461984719848198491985019851198521985319854198551985619857198581985919860198611986219863198641986519866198671986819869198701987119872198731987419875198761987719878198791988019881198821988319884198851988619887198881988919890198911989219893198941989519896198971989819899199001990119902199031990419905199061990719908199091991019911199121991319914199151991619917199181991919920199211992219923199241992519926199271992819929199301993119932199331993419935199361993719938199391994019941199421994319944199451994619947199481994919950199511995219953199541995519956199571995819959199601996119962199631996419965199661996719968199691997019971199721997319974199751997619977199781997919980199811998219983199841998519986199871998819989199901999119992199931999419995199961999719998199992000020001200022000320004200052000620007200082000920010200112001220013200142001520016200172001820019200202002120022200232002420025200262002720028200292003020031200322003320034200352003620037200382003920040200412004220043200442004520046200472004820049200502005120052200532005420055200562005720058200592006020061200622006320064200652006620067// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com * Copyright (c) 2016 Facebook * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io */#include <uapi/linux/btf.h>#include <linux/bpf-cgroup.h>#include <linux/kernel.h>#include <linux/types.h>#include <linux/slab.h>#include <linux/bpf.h>#include <linux/btf.h>#include <linux/bpf_verifier.h>#include <linux/filter.h>#include <net/netlink.h>#include <linux/file.h>#include <linux/vmalloc.h>#include <linux/stringify.h>#include <linux/bsearch.h>#include <linux/sort.h>#include <linux/perf_event.h>#include <linux/ctype.h>#include <linux/error-injection.h>#include <linux/bpf_lsm.h>#include <linux/btf_ids.h>#include <linux/poison.h>#include <linux/module.h>#include <linux/cpumask.h>#include <linux/cnum.h>#include <linux/bpf_mem_alloc.h>#include <net/xdp.h>#include <linux/trace_events.h>#include <linux/kallsyms.h>
#include "disasm.h"
static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ [_id] = & _name ## _verifier_ops,#define BPF_MAP_TYPE(_id, _ops)#define BPF_LINK_TYPE(_id, _name)#include <linux/bpf_types.h>#undef BPF_PROG_TYPE#undef BPF_MAP_TYPE#undef BPF_LINK_TYPE};
enum bpf_features { BPF_FEAT_RDONLY_CAST_TO_VOID = 0, BPF_FEAT_STREAMS = 1, __MAX_BPF_FEAT,};
struct bpf_mem_alloc bpf_global_percpu_ma;static bool bpf_global_percpu_ma_set;
/* bpf_check() is a static code analyzer that walks eBPF program * instruction by instruction and updates register/stack state. * All paths of conditional branches are analyzed until 'bpf_exit' insn. * * The first pass is depth-first-search to check that the program is a DAG. * It rejects the following programs: * - larger than BPF_MAXINSNS insns * - if loop is present (detected via back-edge) * - unreachable insns exist (shouldn't be a forest. program = one function) * - out of bounds or malformed jumps * The second pass is all possible path descent from the 1st insn. * Since it's analyzing all paths through the program, the length of the * analysis is limited to 64k insn, which may be hit even if total number of * insn is less then 4K, but there are too many branches that change stack/regs. * Number of 'branches to be analyzed' is limited to 1k * * On entry to each instruction, each register has a type, and the instruction * changes the types of the registers depending on instruction semantics. * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is * copied to R1. * * All registers are 64-bit. * R0 - return register * R1-R5 argument passing registers * R6-R9 callee saved registers * R10 - frame pointer read-only * * At the start of BPF program the register R1 contains a pointer to bpf_context * and has type PTR_TO_CTX. * * Verifier tracks arithmetic operations on pointers in case: * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), * 1st insn copies R10 (which has FRAME_PTR) type into R1 * and 2nd arithmetic instruction is pattern matched to recognize * that it wants to construct a pointer to some element within stack. * So after 2nd insn, the register R1 has type PTR_TO_STACK * (and -20 constant is saved for further stack bounds checking). * Meaning that this reg is a pointer to stack plus known immediate constant. * * Most of the time the registers have SCALAR_VALUE type, which * means the register has some value, but it's not a valid pointer. * (like pointer plus pointer becomes SCALAR_VALUE type) * * When verifier sees load or store instructions the type of base register * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are * four pointer types recognized by check_mem_access() function. * * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' * and the range of [ptr, ptr + map's value_size) is accessible. * * registers used to pass values to function calls are checked against * function argument constraints. * * ARG_PTR_TO_MAP_KEY is one of such argument constraints. * It means that the register type passed to this function must be * PTR_TO_STACK and it will be used inside the function as * 'pointer to map element key' * * For example the argument constraints for bpf_map_lookup_elem(): * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, * .arg1_type = ARG_CONST_MAP_PTR, * .arg2_type = ARG_PTR_TO_MAP_KEY, * * ret_type says that this function returns 'pointer to map elem value or null' * function expects 1st argument to be a const pointer to 'struct bpf_map' and * 2nd argument should be a pointer to stack, which will be used inside * the helper function as a pointer to map element key. * * On the kernel side the helper function looks like: * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) * { * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; * void *key = (void *) (unsigned long) r2; * void *value; * * here kernel can access 'key' and 'map' pointers safely, knowing that * [key, key + map->key_size) bytes are valid and were initialized on * the stack of eBPF program. * } * * Corresponding eBPF program may look like: * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), * here verifier looks at prototype of map_lookup_elem() and sees: * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, * Now verifier knows that this map has key of R1->map_ptr->key_size bytes * * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, * Now verifier checks that [R2, R2 + map's key_size) are within stack limits * and were initialized prior to this call. * If it's ok, then verifier allows this BPF_CALL insn and looks at * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function * returns either pointer to map value or NULL. * * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' * insn, the register holding that pointer in the true branch changes state to * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false * branch. See check_cond_jmp_op(). * * After the call R0 is set to return type of the function and registers R1-R5 * are set to NOT_INIT to indicate that they are no longer readable. * * The following reference types represent a potential reference to a kernel * resource which, after first being allocated, must be checked and freed by * the BPF program: * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET * * When the verifier sees a helper call return a reference type, it allocates a * pointer id for the reference and stores it in the current function state. * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type * passes through a NULL-check conditional. For the branch wherein the state is * changed to CONST_IMM, the verifier releases the reference. * * For each helper function that allocates a reference, such as * bpf_sk_lookup_tcp(), there is a corresponding release function, such as * bpf_sk_release(). When a reference type passes into the release function, * the verifier also releases the reference. If any unchecked or unreleased * reference remains at the end of the program, the verifier rejects it. */
/* verifier_state + insn_idx are pushed to stack when branch is encountered */struct bpf_verifier_stack_elem { /* verifier state is 'st' * before processing instruction 'insn_idx' * and after processing instruction 'prev_insn_idx' */ struct bpf_verifier_state st; int insn_idx; int prev_insn_idx; struct bpf_verifier_stack_elem *next; /* length of verifier log at the time this state was pushed on stack */ u32 log_pos;};
#define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192#define BPF_COMPLEXITY_LIMIT_STATES 64
#define BPF_GLOBAL_PERCPU_MA_MAX_SIZE 512
#define BPF_PRIV_STACK_MIN_SIZE 64
static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id);static int release_reference_nomark(struct bpf_verifier_state *state, int id);static int release_reference(struct bpf_verifier_env *env, int id);static void invalidate_non_owning_refs(struct bpf_verifier_env *env);static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);static bool is_tracing_prog_type(enum bpf_prog_type type);static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg);static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg);static inline bool in_sleepable_context(struct bpf_verifier_env *env);static const char *non_sleepable_context_description(struct bpf_verifier_env *env);static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg);static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg);
static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, struct bpf_map *map, bool unpriv, bool poison){ unpriv |= bpf_map_ptr_unpriv(aux); aux->map_ptr_state.unpriv = unpriv; aux->map_ptr_state.poison = poison; aux->map_ptr_state.map_ptr = map;}
static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state){ bool poisoned = bpf_map_key_poisoned(aux);
aux->map_key_state = state | BPF_MAP_KEY_SEEN | (poisoned ? BPF_MAP_KEY_POISON : 0ULL);}
static void update_ref_obj(struct ref_obj_desc *ref_obj, struct bpf_reg_state *reg){ ref_obj->id = reg->id; ref_obj->parent_id = reg->parent_id; ref_obj->cnt++;}
static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *ref_obj){ if (ref_obj->cnt > 1) { verifier_bug(env, "function expects only one referenced object but got %d\n", ref_obj->cnt); return -EFAULT; }
return 0;}
struct bpf_call_arg_meta { struct bpf_map_desc map; struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; bool raw_mode; bool pkt_access; u8 release_regno; int regno; int access_size; int mem_size; u64 msize_max_value; int func_id; struct btf *btf; u32 btf_id; struct btf *ret_btf; u32 ret_btf_id; u32 subprogno; struct btf_field *kptr_field; s64 const_map_key;};
struct bpf_kfunc_meta { struct btf *btf; const struct btf_type *proto; const char *name; const u32 *flags; s32 id;};
struct btf *btf_vmlinux;
typedef struct argno { int argno;} argno_t;
static argno_t argno_from_reg(u32 regno){ return (argno_t){ .argno = regno };}
static argno_t argno_from_arg(u32 arg){ return (argno_t){ .argno = -arg };}
static int reg_from_argno(argno_t a){ if (a.argno >= 0) return a.argno; if (a.argno >= -MAX_BPF_FUNC_REG_ARGS) return -a.argno; return -1;}
static int arg_from_argno(argno_t a){ if (a.argno < 0) return -a.argno; return -1;}
static int arg_idx_from_argno(argno_t a){ return arg_from_argno(a) - 1;}
static const char *btf_type_name(const struct btf *btf, u32 id){ return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);}
static DEFINE_MUTEX(bpf_verifier_lock);static DEFINE_MUTEX(bpf_percpu_ma_lock);
__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...){ struct bpf_verifier_env *env = private_data; va_list args;
if (!bpf_verifier_log_needed(&env->log)) return;
va_start(args, fmt); bpf_verifier_vlog(&env->log, fmt, args); va_end(args);}
static void verbose_invalid_scalar(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct bpf_retval_range range, const char *ctx, const char *reg_name){ bool unknown = true;
verbose(env, "%s the register %s has", ctx, reg_name); if (reg_smin(reg) > S64_MIN) { verbose(env, " smin=%lld", reg_smin(reg)); unknown = false; } if (reg_smax(reg) < S64_MAX) { verbose(env, " smax=%lld", reg_smax(reg)); unknown = false; } if (unknown) verbose(env, " unknown scalar value"); verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);}
static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg){ enum bpf_reg_type type;
type = reg->type; if (type_may_be_null(type)) return false;
type = base_type(type); return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK || type == PTR_TO_MAP_VALUE || type == PTR_TO_MAP_KEY || type == PTR_TO_SOCK_COMMON || (type == PTR_TO_BTF_ID && is_trusted_reg(env, reg)) || (type == PTR_TO_MEM && !(reg->type & PTR_UNTRUSTED)) || type == CONST_PTR_TO_MAP;}
static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg){ struct btf_record *rec = NULL; struct btf_struct_meta *meta;
if (reg->type == PTR_TO_MAP_VALUE) { rec = reg->map_ptr->record; } else if (type_is_ptr_alloc_obj(reg->type)) { meta = btf_find_struct_meta(reg->btf, reg->btf_id); if (meta) rec = meta->record; } return rec;}
bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog){ struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;}
static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog){ const struct btf_type *type, *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id;
btf_id = env->prog->aux->func_info[subprog].type_id;
func = btf_type_by_id(btf, btf_id); if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) return false;
func_proto = btf_type_by_id(btf, func->type); if (!func_proto) return false;
type = btf_type_skip_modifiers(btf, func_proto->type, NULL); if (!type) return false;
return btf_type_is_void(type);}
static const char *subprog_name(const struct bpf_verifier_env *env, int subprog){ struct bpf_func_info *info;
if (!env->prog->aux->func_info) return "";
info = &env->prog->aux->func_info[subprog]; return btf_type_name(env->prog->aux->btf, info->type_id);}
void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog){ struct bpf_subprog_info *info = subprog_info(env, subprog);
info->is_cb = true; info->is_async_cb = true; info->is_exception_cb = true;}
static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog){ return subprog_info(env, subprog)->is_exception_cb;}
static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg){ return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK);}
static bool type_is_rdonly_mem(u32 type){ return type & MEM_RDONLY;}
static bool is_acquire_function(enum bpf_func_id func_id, const struct bpf_map *map){ enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
if (func_id == BPF_FUNC_sk_lookup_tcp || func_id == BPF_FUNC_sk_lookup_udp || func_id == BPF_FUNC_skc_lookup_tcp || func_id == BPF_FUNC_ringbuf_reserve || func_id == BPF_FUNC_kptr_xchg) return true;
if (func_id == BPF_FUNC_map_lookup_elem && (map_type == BPF_MAP_TYPE_SOCKMAP || map_type == BPF_MAP_TYPE_SOCKHASH)) return true;
return false;}
static bool is_ptr_cast_function(enum bpf_func_id func_id){ return func_id == BPF_FUNC_tcp_sock || func_id == BPF_FUNC_sk_fullsock || func_id == BPF_FUNC_skc_to_tcp_sock || func_id == BPF_FUNC_skc_to_tcp6_sock || func_id == BPF_FUNC_skc_to_udp6_sock || func_id == BPF_FUNC_skc_to_mptcp_sock || func_id == BPF_FUNC_skc_to_tcp_timewait_sock || func_id == BPF_FUNC_skc_to_tcp_request_sock;}
static bool is_sync_callback_calling_kfunc(u32 btf_id);static bool is_async_callback_calling_kfunc(u32 btf_id);static bool is_callback_calling_kfunc(u32 btf_id);
static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);static bool is_task_work_add_kfunc(u32 func_id);
static bool is_sync_callback_calling_function(enum bpf_func_id func_id){ return func_id == BPF_FUNC_for_each_map_elem || func_id == BPF_FUNC_find_vma || func_id == BPF_FUNC_loop || func_id == BPF_FUNC_user_ringbuf_drain;}
static bool is_async_callback_calling_function(enum bpf_func_id func_id){ return func_id == BPF_FUNC_timer_set_callback;}
static bool is_callback_calling_function(enum bpf_func_id func_id){ return is_sync_callback_calling_function(func_id) || is_async_callback_calling_function(func_id);}
bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn){ return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) || (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));}
bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn){ return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) || (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm));}
static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn){ /* bpf_timer callbacks are never sleepable. */ if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false;
/* bpf_wq and bpf_task_work callbacks are always sleepable. */ if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) return true;
verifier_bug(env, "unhandled async callback in is_async_cb_sleepable"); return false;}
bool bpf_is_may_goto_insn(struct bpf_insn *insn){ return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO;}
static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots){ int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
/* We need to check that slots between [spi - nr_slots + 1, spi] are * within [0, allocated_stack). * * Please note that the spi grows downwards. For example, a dynptr * takes the size of two stack slots; the first slot will be at * spi and the second slot will be at spi - 1. */ return spi - nr_slots + 1 >= 0 && spi < allocated_slots;}
static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *obj_kind, int nr_slots){ int off, spi;
if (!tnum_is_const(reg->var_off)) { verbose(env, "%s has to be at a constant offset\n", obj_kind); return -EINVAL; }
off = reg->var_off.value; if (off % BPF_REG_SIZE) { verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); return -EINVAL; }
spi = bpf_get_spi(off); if (spi + 1 < nr_slots) { verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); return -EINVAL; }
if (!is_spi_bounds_valid(bpf_func(env, reg), spi, nr_slots)) return -ERANGE; return spi;}
static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);}
static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots){ return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);}
static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ return stack_slot_obj_get_spi(env, reg, "irq_flag", 1);}
static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type){ switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { case DYNPTR_TYPE_LOCAL: return BPF_DYNPTR_TYPE_LOCAL; case DYNPTR_TYPE_RINGBUF: return BPF_DYNPTR_TYPE_RINGBUF; case DYNPTR_TYPE_SKB: return BPF_DYNPTR_TYPE_SKB; case DYNPTR_TYPE_XDP: return BPF_DYNPTR_TYPE_XDP; case DYNPTR_TYPE_SKB_META: return BPF_DYNPTR_TYPE_SKB_META; case DYNPTR_TYPE_FILE: return BPF_DYNPTR_TYPE_FILE; default: return BPF_DYNPTR_TYPE_INVALID; }}
static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type){ switch (type) { case BPF_DYNPTR_TYPE_LOCAL: return DYNPTR_TYPE_LOCAL; case BPF_DYNPTR_TYPE_RINGBUF: return DYNPTR_TYPE_RINGBUF; case BPF_DYNPTR_TYPE_SKB: return DYNPTR_TYPE_SKB; case BPF_DYNPTR_TYPE_XDP: return DYNPTR_TYPE_XDP; case BPF_DYNPTR_TYPE_SKB_META: return DYNPTR_TYPE_SKB_META; case BPF_DYNPTR_TYPE_FILE: return DYNPTR_TYPE_FILE; default: return 0; }}
static bool dynptr_type_referenced(enum bpf_dynptr_type type){ return type == BPF_DYNPTR_TYPE_RINGBUF || type == BPF_DYNPTR_TYPE_FILE;}
static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, bool first_slot, int id, int parent_id);
static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, struct bpf_reg_state *sreg1, struct bpf_reg_state *sreg2, enum bpf_dynptr_type type, int parent_id){ int id = ++env->id_gen;
__mark_dynptr_reg(sreg1, type, true, id, parent_id); __mark_dynptr_reg(sreg2, type, false, id, parent_id);}
static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_dynptr_type type){ __mark_dynptr_reg(reg, type, true, ++env->id_gen, 0);}
static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi);
static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type, int insn_idx, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr){ struct bpf_func_state *state = bpf_func(env, reg); int spi, i, err, parent_id = 0; enum bpf_dynptr_type type;
spi = dynptr_get_spi(env, reg); if (spi < 0) return spi;
/* We cannot assume both spi and spi - 1 belong to the same dynptr, * hence we need to call destroy_if_dynptr_stack_slot twice for both, * to ensure that for the following example: * [d1][d1][d2][d2] * spi 3 2 1 0 * So marking spi = 2 should lead to destruction of both d1 and d2. In * case they do belong to same dynptr, second call won't see slot_type * as STACK_DYNPTR and will simply skip destruction. */ err = destroy_if_dynptr_stack_slot(env, state, spi); if (err) return err; err = destroy_if_dynptr_stack_slot(env, state, spi - 1); if (err) return err;
for (i = 0; i < BPF_REG_SIZE; i++) { state->stack[spi].slot_type[i] = STACK_DYNPTR; state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; }
type = arg_to_dynptr_type(arg_type); if (type == BPF_DYNPTR_TYPE_INVALID) return -EINVAL;
if (dynptr->type == BPF_DYNPTR_TYPE_INVALID) { /* dynptr constructors */ err = validate_ref_obj(env, ref_obj); if (err) return err;
/* Track parent's id if the parent is a referenced object */ parent_id = ref_obj->id;
if (dynptr_type_referenced(type)) { int id;
/* * Create an intermediate reference that tracks the referenced * object for the referenced dynptr. Freeing a referenced dynptr * through helpers/kfuncs will invalidate all clones. */ id = acquire_reference(env, insn_idx, parent_id); if (id < 0) return id;
parent_id = id; } } else { /* bpf_dynptr_clone() */ parent_id = dynptr->parent_id; }
mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, &state->stack[spi - 1].spilled_ptr, type, parent_id);
return 0;}
static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_stack_state *stack){ int i;
for (i = 0; i < BPF_REG_SIZE; i++) { stack[0].slot_type[i] = STACK_INVALID; stack[1].slot_type[i] = STACK_INVALID; }
bpf_mark_reg_not_init(env, &stack[0].spilled_ptr); bpf_mark_reg_not_init(env, &stack[1].spilled_ptr);}
static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ struct bpf_func_state *state = bpf_func(env, reg); int spi;
spi = dynptr_get_spi(env, reg); if (spi < 0) return spi;
/* * For referenced dynptr, release the parent ref which cascades to * all clones and derived slices. For non-referenced dynptr, only * the dynptr and slices derived from it will be invalidated. */ reg = &state->stack[spi].spilled_ptr; return release_reference(env, dynptr_type_referenced(reg->dynptr.type) ? reg->parent_id : reg->id);}
static void __mark_reg_unknown(const struct bpf_verifier_env *env, struct bpf_reg_state *reg);
static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg){ if (!env->allow_ptr_leaks) bpf_mark_reg_not_init(env, reg); else __mark_reg_unknown(env, reg);}
static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id){ struct bpf_stack_state *stack; struct bpf_func_state *state; struct bpf_reg_state *reg; int ref_cnt = 0;
bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, 1 << STACK_DYNPTR, ({ if (!stack || stack->slot_type[0] != STACK_DYNPTR) continue; if (!stack->spilled_ptr.dynptr.first_slot) continue; if (stack->spilled_ptr.parent_id == v_parent_id) ref_cnt++; }));
return ref_cnt;}
static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi){ int err = 0;
/* We always ensure that STACK_DYNPTR is never set partially, * hence just checking for slot_type[0] is enough. This is * different for STACK_SPILL, where it may be only set for * 1 byte, so code has to use is_spilled_reg. */ if (state->stack[spi].slot_type[0] != STACK_DYNPTR) return 0;
/* Reposition spi to first slot */ if (!state->stack[spi].spilled_ptr.dynptr.first_slot) spi = spi + 1;
/* * A referenced dynptr can be overwritten only if there is at * least one other dynptr sharing the same virtual ref parent, * ensuring the reference can still be properly released. */ if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); return -EINVAL; }
/* Invalidate the dynptr and any derived slices */ err = release_reference(env, state->stack[spi].spilled_ptr.id); if (!err) { mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi - 1); }
return err;}
static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ int spi;
if (reg->type == CONST_PTR_TO_DYNPTR) return false;
spi = dynptr_get_spi(env, reg);
/* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an * error because this just means the stack state hasn't been updated yet. * We will do check_mem_access to check and update stack bounds later. */ if (spi < 0 && spi != -ERANGE) return false;
/* We don't need to check if the stack slots are marked by previous * dynptr initializations because we allow overwriting existing unreferenced * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are * touching are completely destructed before we reinitialize them for a new * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early * instead of delaying it until the end where the user will get "Unreleased * reference" error. */ return true;}
static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ struct bpf_func_state *state = bpf_func(env, reg); int i, spi;
/* This already represents first slot of initialized bpf_dynptr. * * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to * check_func_arg_reg_off's logic, so we don't need to check its * offset and alignment. */ if (reg->type == CONST_PTR_TO_DYNPTR) return true;
spi = dynptr_get_spi(env, reg); if (spi < 0) return false; if (!state->stack[spi].spilled_ptr.dynptr.first_slot) return false;
for (i = 0; i < BPF_REG_SIZE; i++) { if (state->stack[spi].slot_type[i] != STACK_DYNPTR || state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) return false; }
return true;}
static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type){ struct bpf_func_state *state = bpf_func(env, reg); enum bpf_dynptr_type dynptr_type; int spi;
/* ARG_PTR_TO_DYNPTR takes any type of dynptr */ if (arg_type == ARG_PTR_TO_DYNPTR) return true;
dynptr_type = arg_to_dynptr_type(arg_type); if (reg->type == CONST_PTR_TO_DYNPTR) { return reg->dynptr.type == dynptr_type; } else { spi = dynptr_get_spi(env, reg); if (spi < 0) return false; return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; }}
static void __mark_reg_known_zero(struct bpf_reg_state *reg);
static bool in_rcu_cs(struct bpf_verifier_env *env);
static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta);
static int mark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, struct btf *btf, u32 btf_id, int nr_slots){ struct bpf_func_state *state = bpf_func(env, reg); int spi, i, j, id;
spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) return spi;
id = acquire_reference(env, insn_idx, 0); if (id < 0) return id;
for (i = 0; i < nr_slots; i++) { struct bpf_stack_state *slot = &state->stack[spi - i]; struct bpf_reg_state *st = &slot->spilled_ptr;
__mark_reg_known_zero(st); st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ if (is_kfunc_rcu_protected(meta)) { if (in_rcu_cs(env)) st->type |= MEM_RCU; else st->type |= PTR_UNTRUSTED; } st->id = i == 0 ? id : 0; st->iter.btf = btf; st->iter.btf_id = btf_id; st->iter.state = BPF_ITER_STATE_ACTIVE; st->iter.depth = 0;
for (j = 0; j < BPF_REG_SIZE; j++) slot->slot_type[j] = STACK_ITER;
mark_stack_slot_scratched(env, spi - i); }
return 0;}
static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots){ struct bpf_func_state *state = bpf_func(env, reg); int spi, i, j;
spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) return spi;
for (i = 0; i < nr_slots; i++) { struct bpf_stack_state *slot = &state->stack[spi - i]; struct bpf_reg_state *st = &slot->spilled_ptr;
if (i == 0) WARN_ON_ONCE(release_reference(env, st->id));
bpf_mark_reg_not_init(env, st);
for (j = 0; j < BPF_REG_SIZE; j++) slot->slot_type[j] = STACK_INVALID;
mark_stack_slot_scratched(env, spi - i); }
return 0;}
static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots){ struct bpf_func_state *state = bpf_func(env, reg); int spi, i, j;
/* For -ERANGE (i.e. spi not falling into allocated stack slots), we * will do check_mem_access to check and update stack bounds later, so * return true for that case. */ spi = iter_get_spi(env, reg, nr_slots); if (spi == -ERANGE) return true; if (spi < 0) return false;
for (i = 0; i < nr_slots; i++) { struct bpf_stack_state *slot = &state->stack[spi - i];
for (j = 0; j < BPF_REG_SIZE; j++) if (slot->slot_type[j] == STACK_ITER) return false; }
return true;}
static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct btf *btf, u32 btf_id, int nr_slots){ struct bpf_func_state *state = bpf_func(env, reg); int spi, i, j;
spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) return -EINVAL;
for (i = 0; i < nr_slots; i++) { struct bpf_stack_state *slot = &state->stack[spi - i]; struct bpf_reg_state *st = &slot->spilled_ptr;
if (st->type & PTR_UNTRUSTED) return -EPROTO; /* only main (first) slot has id set */ if (i == 0 && !st->id) return -EINVAL; if (i != 0 && st->id) return -EINVAL; if (st->iter.btf != btf || st->iter.btf_id != btf_id) return -EINVAL;
for (j = 0; j < BPF_REG_SIZE; j++) if (slot->slot_type[j] != STACK_ITER) return -EINVAL; }
return 0;}
static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);static int release_irq_state(struct bpf_verifier_state *state, int id);
static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, int kfunc_class){ struct bpf_func_state *state = bpf_func(env, reg); struct bpf_stack_state *slot; struct bpf_reg_state *st; int spi, i, id;
spi = irq_flag_get_spi(env, reg); if (spi < 0) return spi;
id = acquire_irq_state(env, insn_idx); if (id < 0) return id;
slot = &state->stack[spi]; st = &slot->spilled_ptr;
__mark_reg_known_zero(st); st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ st->id = id; st->irq.kfunc_class = kfunc_class;
for (i = 0; i < BPF_REG_SIZE; i++) slot->slot_type[i] = STACK_IRQ_FLAG;
mark_stack_slot_scratched(env, spi); return 0;}
static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int kfunc_class){ struct bpf_func_state *state = bpf_func(env, reg); struct bpf_stack_state *slot; struct bpf_reg_state *st; int spi, i, err;
spi = irq_flag_get_spi(env, reg); if (spi < 0) return spi;
slot = &state->stack[spi]; st = &slot->spilled_ptr;
if (st->irq.kfunc_class != kfunc_class) { const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock";
verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", flag_kfunc, used_kfunc); return -EINVAL; }
err = release_irq_state(env->cur_state, st->id); WARN_ON_ONCE(err && err != -EACCES); if (err) { int insn_idx = 0;
for (int i = 0; i < env->cur_state->acquired_refs; i++) { if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) { insn_idx = env->cur_state->refs[i].insn_idx; break; } }
verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", env->cur_state->active_irq_id, insn_idx); return err; }
bpf_mark_reg_not_init(env, st);
for (i = 0; i < BPF_REG_SIZE; i++) slot->slot_type[i] = STACK_INVALID;
mark_stack_slot_scratched(env, spi); return 0;}
static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ struct bpf_func_state *state = bpf_func(env, reg); struct bpf_stack_state *slot; int spi, i;
/* For -ERANGE (i.e. spi not falling into allocated stack slots), we * will do check_mem_access to check and update stack bounds later, so * return true for that case. */ spi = irq_flag_get_spi(env, reg); if (spi == -ERANGE) return true; if (spi < 0) return false;
slot = &state->stack[spi];
for (i = 0; i < BPF_REG_SIZE; i++) if (slot->slot_type[i] == STACK_IRQ_FLAG) return false; return true;}
static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ struct bpf_func_state *state = bpf_func(env, reg); struct bpf_stack_state *slot; struct bpf_reg_state *st; int spi, i;
spi = irq_flag_get_spi(env, reg); if (spi < 0) return -EINVAL;
slot = &state->stack[spi]; st = &slot->spilled_ptr;
if (!st->id) return -EINVAL;
for (i = 0; i < BPF_REG_SIZE; i++) if (slot->slot_type[i] != STACK_IRQ_FLAG) return -EINVAL; return 0;}
/* Check if given stack slot is "special": * - spilled register state (STACK_SPILL); * - dynptr state (STACK_DYNPTR); * - iter state (STACK_ITER). * - irq flag state (STACK_IRQ_FLAG) */static bool is_stack_slot_special(const struct bpf_stack_state *stack){ enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1];
switch (type) { case STACK_SPILL: case STACK_DYNPTR: case STACK_ITER: case STACK_IRQ_FLAG: return true; case STACK_INVALID: case STACK_POISON: case STACK_MISC: case STACK_ZERO: return false; default: WARN_ONCE(1, "unknown stack slot type %d\n", type); return true; }}
/* The reg state of a pointer or a bounded scalar was saved when * it was spilled to the stack. */
/* * Mark stack slot as STACK_MISC, unless it is already: * - STACK_INVALID, in which case they are equivalent. * - STACK_ZERO, in which case we preserve more precise STACK_ZERO. * - STACK_POISON, which truly forbids access to the slot. * Regardless of allow_ptr_leaks setting (i.e., privileged or unprivileged * mode), we won't promote STACK_INVALID to STACK_MISC. In privileged case it is * unnecessary as both are considered equivalent when loading data and pruning, * in case of unprivileged mode it will be incorrect to allow reads of invalid * slots. */static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype){ if (*stype == STACK_ZERO) return; if (*stype == STACK_INVALID || *stype == STACK_POISON) return; *stype = STACK_MISC;}
static void scrub_spilled_slot(u8 *stype){ if (*stype != STACK_INVALID && *stype != STACK_POISON) *stype = STACK_MISC;}
/* copy array src of length n * size bytes to dst. dst is reallocated if it's too * small to hold src. This is different from krealloc since we don't want to preserve * the contents of dst. * * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could * not be allocated. */static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags){ size_t alloc_bytes; void *orig = dst; size_t bytes;
if (ZERO_OR_NULL_PTR(src)) goto out;
if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL;
alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes)); dst = krealloc(orig, alloc_bytes, flags); if (!dst) { kfree(orig); return NULL; }
memcpy(dst, src, bytes);out: return dst ? dst : ZERO_SIZE_PTR;}
/* resize an array from old_n items to new_n items. the array is reallocated if it's too * small to hold new_n items. new items are zeroed out if the array grows. * * Contrary to krealloc_array, does not free arr if new_n is zero. */static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size){ size_t alloc_size; void *new_arr;
if (!new_n || old_n == new_n) goto out;
alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); new_arr = krealloc(arr, alloc_size, GFP_KERNEL_ACCOUNT); if (!new_arr) { kfree(arr); return NULL; } arr = new_arr;
if (new_n > old_n) memset(arr + old_n * size, 0, (new_n - old_n) * size);
out: return arr ? arr : ZERO_SIZE_PTR;}
static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src){ dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, sizeof(struct bpf_reference_state), GFP_KERNEL_ACCOUNT); if (!dst->refs) return -ENOMEM;
dst->acquired_refs = src->acquired_refs; dst->active_locks = src->active_locks; dst->active_preempt_locks = src->active_preempt_locks; dst->active_rcu_locks = src->active_rcu_locks; dst->active_irq_id = src->active_irq_id; dst->active_lock_id = src->active_lock_id; dst->active_lock_ptr = src->active_lock_ptr; return 0;}
static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src){ size_t n = src->allocated_stack / BPF_REG_SIZE;
dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), GFP_KERNEL_ACCOUNT); if (!dst->stack) return -ENOMEM;
dst->allocated_stack = src->allocated_stack;
/* copy stack args state */ n = src->out_stack_arg_cnt; if (n) { dst->stack_arg_regs = copy_array(dst->stack_arg_regs, src->stack_arg_regs, n, sizeof(struct bpf_reg_state), GFP_KERNEL_ACCOUNT); if (!dst->stack_arg_regs) return -ENOMEM; }
dst->out_stack_arg_cnt = src->out_stack_arg_cnt; return 0;}
static int resize_reference_state(struct bpf_verifier_state *state, size_t n){ state->refs = realloc_array(state->refs, state->acquired_refs, n, sizeof(struct bpf_reference_state)); if (!state->refs) return -ENOMEM;
state->acquired_refs = n; return 0;}
/* Possibly update state->allocated_stack to be at least size bytes. Also * possibly update the function's high-water mark in its bpf_subprog_info. */static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size){ size_t old_n = state->allocated_stack / BPF_REG_SIZE, n;
/* The stack size is always a multiple of BPF_REG_SIZE. */ size = round_up(size, BPF_REG_SIZE); n = size / BPF_REG_SIZE;
if (old_n >= n) return 0;
state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); if (!state->stack) return -ENOMEM;
state->allocated_stack = size;
/* update known max for given subprogram */ if (env->subprog_info[state->subprogno].stack_depth < size) env->subprog_info[state->subprogno].stack_depth = size;
return 0;}
static int grow_stack_arg_slots(struct bpf_verifier_env *env, struct bpf_func_state *state, int cnt){ size_t old_n = state->out_stack_arg_cnt;
if (old_n >= cnt) return 0;
state->stack_arg_regs = realloc_array(state->stack_arg_regs, old_n, cnt, sizeof(struct bpf_reg_state)); if (!state->stack_arg_regs) return -ENOMEM;
state->out_stack_arg_cnt = cnt; return 0;}
/* Acquire a pointer id from the env and update the state->refs to include * this new pointer reference. * On success, returns a valid pointer id to associate with the register * On failure, returns a negative errno. */static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx){ struct bpf_verifier_state *state = env->cur_state; int new_ofs = state->acquired_refs; int err;
err = resize_reference_state(state, state->acquired_refs + 1); if (err) return NULL; state->refs[new_ofs].insn_idx = insn_idx;
return &state->refs[new_ofs];}
static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id){ struct bpf_reference_state *s;
s = acquire_reference_state(env, insn_idx); if (!s) return -ENOMEM; s->type = REF_TYPE_PTR; s->id = ++env->id_gen; s->parent_id = parent_id; return s->id;}
static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type, int id, void *ptr){ struct bpf_verifier_state *state = env->cur_state; struct bpf_reference_state *s;
s = acquire_reference_state(env, insn_idx); if (!s) return -ENOMEM; s->type = type; s->id = id; s->ptr = ptr;
state->active_locks++; state->active_lock_id = id; state->active_lock_ptr = ptr; return 0;}
static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx){ struct bpf_verifier_state *state = env->cur_state; struct bpf_reference_state *s;
s = acquire_reference_state(env, insn_idx); if (!s) return -ENOMEM; s->type = REF_TYPE_IRQ; s->id = ++env->id_gen;
state->active_irq_id = s->id; return s->id;}
static void release_reference_state(struct bpf_verifier_state *state, int idx){ int last_idx; size_t rem;
/* IRQ state requires the relative ordering of elements remaining the * same, since it relies on the refs array to behave as a stack, so that * it can detect out-of-order IRQ restore. Hence use memmove to shift * the array instead of swapping the final element into the deleted idx. */ last_idx = state->acquired_refs - 1; rem = state->acquired_refs - idx - 1; if (last_idx && idx != last_idx) memmove(&state->refs[idx], &state->refs[idx + 1], sizeof(*state->refs) * rem); memset(&state->refs[last_idx], 0, sizeof(*state->refs)); state->acquired_refs--; return;}
static bool find_reference_state(struct bpf_verifier_state *state, int id){ int i;
for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].type != REF_TYPE_PTR) continue; if (state->refs[i].id == id) return true; }
return false;}
static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg_state *reg){ return find_reference_state(env->cur_state, reg->id);}
static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr){ void *prev_ptr = NULL; u32 prev_id = 0; int i;
for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].type == type && state->refs[i].id == id && state->refs[i].ptr == ptr) { release_reference_state(state, i); state->active_locks--; /* Reassign active lock (id, ptr). */ state->active_lock_id = prev_id; state->active_lock_ptr = prev_ptr; return 0; } if (state->refs[i].type & REF_TYPE_LOCK_MASK) { prev_id = state->refs[i].id; prev_ptr = state->refs[i].ptr; } } return -EINVAL;}
static int release_irq_state(struct bpf_verifier_state *state, int id){ u32 prev_id = 0; int i;
if (id != state->active_irq_id) return -EACCES;
for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].type != REF_TYPE_IRQ) continue; if (state->refs[i].id == id) { release_reference_state(state, i); state->active_irq_id = prev_id; return 0; } else { prev_id = state->refs[i].id; } } return -EINVAL;}
static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *state, enum ref_state_type type, int id, void *ptr){ int i;
for (i = 0; i < state->acquired_refs; i++) { struct bpf_reference_state *s = &state->refs[i];
if (!(s->type & type)) continue;
if (s->id == id && s->ptr == ptr) return s; } return NULL;}
static void free_func_state(struct bpf_func_state *state){ if (!state) return; kfree(state->stack_arg_regs); kfree(state->stack); kfree(state);}
void bpf_clear_jmp_history(struct bpf_verifier_state *state){ kfree(state->jmp_history); state->jmp_history = NULL; state->jmp_history_cnt = 0;}
void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self){ int i;
for (i = 0; i <= state->curframe; i++) { free_func_state(state->frame[i]); state->frame[i] = NULL; } kfree(state->refs); bpf_clear_jmp_history(state); if (free_self) kfree(state);}
/* copy verifier state from src to dst growing dst stack space * when necessary to accommodate larger src stack */static int copy_func_state(struct bpf_func_state *dst, const struct bpf_func_state *src){ memcpy(dst, src, offsetof(struct bpf_func_state, stack)); return copy_stack_state(dst, src);}
int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, const struct bpf_verifier_state *src){ struct bpf_func_state *dst; int i, err;
dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, src->jmp_history_cnt, sizeof(*dst_state->jmp_history), GFP_KERNEL_ACCOUNT); if (!dst_state->jmp_history) return -ENOMEM; dst_state->jmp_history_cnt = src->jmp_history_cnt;
/* if dst has more stack frames then src frame, free them, this is also * necessary in case of exceptional exits using bpf_throw. */ for (i = src->curframe + 1; i <= dst_state->curframe; i++) { free_func_state(dst_state->frame[i]); dst_state->frame[i] = NULL; } err = copy_reference_state(dst_state, src); if (err) return err; dst_state->speculative = src->speculative; dst_state->in_sleepable = src->in_sleepable; dst_state->curframe = src->curframe; dst_state->branches = src->branches; dst_state->parent = src->parent; dst_state->first_insn_idx = src->first_insn_idx; dst_state->last_insn_idx = src->last_insn_idx; dst_state->dfs_depth = src->dfs_depth; dst_state->callback_unroll_depth = src->callback_unroll_depth; dst_state->may_goto_depth = src->may_goto_depth; dst_state->equal_state = src->equal_state; for (i = 0; i <= src->curframe; i++) { dst = dst_state->frame[i]; if (!dst) { dst = kzalloc_obj(*dst, GFP_KERNEL_ACCOUNT); if (!dst) return -ENOMEM; dst_state->frame[i] = dst; } err = copy_func_state(dst, src->frame[i]); if (err) return err; } return 0;}
static u32 state_htab_size(struct bpf_verifier_env *env){ return env->prog->len;}
struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx){ struct bpf_verifier_state *cur = env->cur_state; struct bpf_func_state *state = cur->frame[cur->curframe];
return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];}
static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b){ int fr;
if (a->curframe != b->curframe) return false;
for (fr = a->curframe; fr >= 0; fr--) if (a->frame[fr]->callsite != b->frame[fr]->callsite) return false;
return true;}
void bpf_free_backedges(struct bpf_scc_visit *visit){ struct bpf_scc_backedge *backedge, *next;
for (backedge = visit->backedges; backedge; backedge = next) { bpf_free_verifier_state(&backedge->state, false); next = backedge->next; kfree(backedge); } visit->backedges = NULL;}
static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, int *insn_idx, bool pop_log){ struct bpf_verifier_state *cur = env->cur_state; struct bpf_verifier_stack_elem *elem, *head = env->head; int err;
if (env->head == NULL) return -ENOENT;
if (cur) { err = bpf_copy_verifier_state(cur, &head->st); if (err) return err; } if (pop_log) bpf_vlog_reset(&env->log, head->log_pos); if (insn_idx) *insn_idx = head->insn_idx; if (prev_insn_idx) *prev_insn_idx = head->prev_insn_idx; elem = head->next; bpf_free_verifier_state(&head->st, false); kfree(head); env->head = elem; env->stack_size--; return 0;}
static bool error_recoverable_with_nospec(int err){ /* Should only return true for non-fatal errors that are allowed to * occur during speculative verification. For these we can insert a * nospec and the program might still be accepted. Do not include * something like ENOMEM because it is likely to re-occur for the next * architectural path once it has been recovered-from in all speculative * paths. */ return err == -EPERM || err == -EACCES || err == -EINVAL;}
static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx, bool speculative){ struct bpf_verifier_state *cur = env->cur_state; struct bpf_verifier_stack_elem *elem; int err;
elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT); if (!elem) return ERR_PTR(-ENOMEM);
elem->insn_idx = insn_idx; elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; env->head = elem; env->stack_size++; err = bpf_copy_verifier_state(&elem->st, cur); if (err) return ERR_PTR(-ENOMEM); elem->st.speculative |= speculative; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { verbose(env, "The sequence of %d jumps is too complex.\n", env->stack_size); return ERR_PTR(-E2BIG); } if (elem->st.parent) { ++elem->st.parent->branches; /* WARN_ON(branches > 2) technically makes sense here, * but * 1. speculative states will bump 'branches' for non-branch * instructions * 2. is_state_visited() heuristics may decide not to create * a new state for a sequence of branches and all such current * and cloned states will be pointing to a single parent state * which might have large 'branches' count. */ } return &elem->st;}
static const char *reg_arg_name(struct bpf_verifier_env *env, argno_t argno){ char *buf = env->tmp_arg_name; int len = sizeof(env->tmp_arg_name); int arg, regno = reg_from_argno(argno);
if (regno >= 0) { snprintf(buf, len, "R%d", regno); } else { arg = arg_from_argno(argno); snprintf(buf, len, "*(R11-%u)", (arg - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE); }
return buf;}
static const int caller_saved[CALLER_SAVED_REGS] = { BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5};
/* This helper doesn't clear reg->id */static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm){ reg->var_off = tnum_const(imm); reg->r64 = cnum64_from_urange(imm, imm); reg->r32 = cnum32_from_urange((u32)imm, (u32)imm);}
/* Mark the unknown part of a register (variable offset or scalar value) as * known to have the value @imm. */static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm){ /* Clear off and union(map_ptr, range) */ memset(((u8 *)reg) + sizeof(reg->type), 0, offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); reg->id = 0; reg->parent_id = 0; ___mark_reg_known(reg, imm);}
static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm){ reg->var_off = tnum_const_subreg(reg->var_off, imm); reg->r32 = cnum32_from_urange((u32)imm, (u32)imm);}
/* Mark the 'variable offset' part of a register as zero. This should be * used only on registers holding a pointer type. */static void __mark_reg_known_zero(struct bpf_reg_state *reg){ __mark_reg_known(reg, 0);}
static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg){ __mark_reg_known(reg, 0); reg->type = SCALAR_VALUE; /* all scalars are assumed imprecise initially (unless unprivileged, * in which case everything is forced to be precise) */ reg->precise = !env->bpf_capable;}
static void mark_reg_known_zero(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno){ __mark_reg_known_zero(regs + regno);}
static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, bool first_slot, int id, int parent_id){ /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply * set it unconditionally as it is ignored for STACK_DYNPTR anyway. */ __mark_reg_known_zero(reg); reg->type = CONST_PTR_TO_DYNPTR; /* Give each dynptr a unique id to uniquely associate slices to it. */ reg->id = id; reg->parent_id = parent_id; reg->dynptr.type = type; reg->dynptr.first_slot = first_slot;}
static void mark_ptr_not_null_reg(struct bpf_reg_state *reg){ if (base_type(reg->type) == PTR_TO_MAP_VALUE) { const struct bpf_map *map = reg->map_ptr;
if (map->inner_map_meta) { reg->type = CONST_PTR_TO_MAP; reg->map_ptr = map->inner_map_meta; /* transfer reg's id which is unique for every map_lookup_elem * as UID of the inner map. */ if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) { reg->map_uid = reg->id; } } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { reg->type = PTR_TO_XDP_SOCK; } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || map->map_type == BPF_MAP_TYPE_SOCKHASH) { reg->type = PTR_TO_SOCKET; } else { reg->type = PTR_TO_MAP_VALUE; } return; }
reg->type &= ~PTR_MAYBE_NULL;}
static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno, struct btf_field_graph_root *ds_head){ __mark_reg_known(®s[regno], ds_head->node_offset); regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC; regs[regno].btf = ds_head->btf; regs[regno].btf_id = ds_head->value_btf_id;}
static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg){ return type_is_pkt_pointer(reg->type);}
static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg){ return reg_is_pkt_pointer(reg) || reg->type == PTR_TO_PACKET_END;}
static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg){ return base_type(reg->type) == PTR_TO_MEM && (reg->type & (DYNPTR_TYPE_SKB | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META));}
/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, enum bpf_reg_type which){ /* The register can already have a range from prior markings. * This is fine as long as it hasn't been advanced from its * origin. */ return reg->type == which && reg->id == 0 && tnum_equals_const(reg->var_off, 0);}
static void __mark_reg32_unbounded(struct bpf_reg_state *reg){ reg->r32 = CNUM32_UNBOUNDED;}
static void __mark_reg64_unbounded(struct bpf_reg_state *reg){ reg->r64 = CNUM64_UNBOUNDED;}
/* Reset the min/max bounds of a register */static void __mark_reg_unbounded(struct bpf_reg_state *reg){ __mark_reg64_unbounded(reg); __mark_reg32_unbounded(reg);}
static void reset_reg64_and_tnum(struct bpf_reg_state *reg){ __mark_reg64_unbounded(reg); reg->var_off = tnum_unknown;}
static void reset_reg32_and_tnum(struct bpf_reg_state *reg){ __mark_reg32_unbounded(reg); reg->var_off = tnum_unknown;}
static struct cnum32 cnum32_from_tnum(struct tnum tnum){ tnum = tnum_subreg(tnum); if ((tnum.mask & S32_MIN) || (tnum.value & S32_MIN)) /* min signed is max(sign bit) | min(other bits) */ /* max signed is min(sign bit) | max(other bits) */ return cnum32_from_srange(tnum.value | (tnum.mask & S32_MIN), tnum.value | (tnum.mask & S32_MAX)); else return cnum32_from_urange(tnum.value, (tnum.value | tnum.mask));}
static struct cnum64 cnum64_from_tnum(struct tnum tnum){ if ((tnum.mask & S64_MIN) || (tnum.value & S64_MIN)) /* min signed is max(sign bit) | min(other bits) */ /* max signed is min(sign bit) | max(other bits) */ return cnum64_from_srange(tnum.value | (tnum.mask & S64_MIN), tnum.value | (tnum.mask & S64_MAX)); else return cnum64_from_urange(tnum.value, (tnum.value | tnum.mask));}
static void __update_reg32_bounds(struct bpf_reg_state *reg){ cnum32_intersect_with(®->r32, cnum32_from_tnum(reg->var_off));}
static void __update_reg64_bounds(struct bpf_reg_state *reg){ u64 tnum_next, tmax; bool umin_in_tnum;
cnum64_intersect_with(®->r64, cnum64_from_tnum(reg->var_off));
/* Check if u64 and tnum overlap in a single value */ tnum_next = tnum_step(reg->var_off, reg_umin(reg)); umin_in_tnum = (reg_umin(reg) & ~reg->var_off.mask) == reg->var_off.value; tmax = reg->var_off.value | reg->var_off.mask; if (umin_in_tnum && tnum_next > reg_umax(reg)) { /* The u64 range and the tnum only overlap in umin. * u64: ---[xxxxxx]----- * tnum: --xx----------x- */ ___mark_reg_known(reg, reg_umin(reg)); } else if (!umin_in_tnum && tnum_next == tmax) { /* The u64 range and the tnum only overlap in the maximum value * represented by the tnum, called tmax. * u64: ---[xxxxxx]----- * tnum: xx-----x-------- */ ___mark_reg_known(reg, tmax); } else if (!umin_in_tnum && tnum_next <= reg_umax(reg) && tnum_step(reg->var_off, tnum_next) > reg_umax(reg)) { /* The u64 range and the tnum only overlap in between umin * (excluded) and umax. * u64: ---[xxxxxx]----- * tnum: xx----x-------x- */ ___mark_reg_known(reg, tnum_next); }}
static void __update_reg_bounds(struct bpf_reg_state *reg){ __update_reg32_bounds(reg); __update_reg64_bounds(reg);}
static void deduce_bounds_32_from_64(struct bpf_reg_state *reg){ cnum32_intersect_with(®->r32, cnum32_from_cnum64(reg->r64));}
static void deduce_bounds_64_from_32(struct bpf_reg_state *reg){ reg->r64 = cnum64_cnum32_intersect(reg->r64, reg->r32);}
static void __reg_deduce_bounds(struct bpf_reg_state *reg){ deduce_bounds_32_from_64(reg); deduce_bounds_64_from_32(reg);}
/* Attempts to improve var_off based on unsigned min/max information */static void __reg_bound_offset(struct bpf_reg_state *reg){ struct tnum var64_off = tnum_intersect(reg->var_off, tnum_range(reg_umin(reg), reg_umax(reg))); struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), tnum_range(reg_u32_min(reg), reg_u32_max(reg)));
reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);}
static bool range_bounds_violation(struct bpf_reg_state *reg);
static void reg_bounds_sync(struct bpf_reg_state *reg){ /* If the input reg_state is invalid, we can exit early */ if (range_bounds_violation(reg)) return; /* We might have learned new bounds from the var_off. */ __update_reg_bounds(reg); /* We might have learned something about the sign bit. */ __reg_deduce_bounds(reg); __reg_deduce_bounds(reg); /* We might have learned some bits from the bounds. */ __reg_bound_offset(reg); /* Intersecting with the old var_off might have improved our bounds * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), * then new var_off is (0; 0x7f...fc) which improves our umax. */ __update_reg_bounds(reg);}
static bool const_tnum_range_mismatch(struct bpf_reg_state *reg){ if (!tnum_is_const(reg->var_off)) return false;
return !cnum64_is_const(reg->r64) || reg->r64.base != reg->var_off.value;}
static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg){ if (!tnum_subreg_is_const(reg->var_off)) return false;
return !cnum32_is_const(reg->r32) || reg->r32.base != tnum_subreg(reg->var_off).value;}
static bool range_bounds_violation(struct bpf_reg_state *reg){ return cnum32_is_empty(reg->r32) || cnum64_is_empty(reg->r64);}
static int reg_bounds_sanity_check(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *ctx){ const char *msg;
if (range_bounds_violation(reg)) { msg = "range bounds violation"; goto out; }
if (const_tnum_range_mismatch(reg)) { msg = "const tnum out of sync with range bounds"; goto out; }
if (const_tnum_range_mismatch_32(reg)) { msg = "const subreg tnum out of sync with range bounds"; goto out; }
return 0;out: verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)", ctx, msg, reg->r64.base, reg->r64.size, reg->r32.base, reg->r32.size, reg->var_off.value, reg->var_off.mask); if (env->test_reg_invariants) return -EFAULT; __mark_reg_unbounded(reg); return 0;}
/* Mark a register as having a completely unknown (scalar) value. */void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg){ s32 subreg_def = reg->subreg_def;
memset(reg, 0, sizeof(*reg)); reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; reg->subreg_def = subreg_def; __mark_reg_unbounded(reg);}
/* Mark a register as having a completely unknown (scalar) value, * initialize .precise as true when not bpf capable. */static void __mark_reg_unknown(const struct bpf_verifier_env *env, struct bpf_reg_state *reg){ bpf_mark_reg_unknown_imprecise(reg); reg->precise = !env->bpf_capable;}
static void mark_reg_unknown(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno){ __mark_reg_unknown(env, regs + regno);}
static int __mark_reg_s32_range(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, s32 s32_min, s32 s32_max){ struct bpf_reg_state *reg = regs + regno;
reg_set_srange32(reg, max_t(s32, reg_s32_min(reg), s32_min), min_t(s32, reg_s32_max(reg), s32_max)); reg_set_srange64(reg, max_t(s64, reg_smin(reg), s32_min), min_t(s64, reg_smax(reg), s32_max));
reg_bounds_sync(reg);
return reg_bounds_sanity_check(env, reg, "s32_range");}
void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg){ __mark_reg_unknown(env, reg); reg->type = NOT_INIT;}
static int mark_btf_ld_reg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, enum bpf_reg_type reg_type, struct btf *btf, u32 btf_id, enum bpf_type_flag flag){ switch (reg_type) { case SCALAR_VALUE: mark_reg_unknown(env, regs, regno); return 0; case PTR_TO_BTF_ID: mark_reg_known_zero(env, regs, regno); regs[regno].type = PTR_TO_BTF_ID | flag; regs[regno].btf = btf; regs[regno].btf_id = btf_id; if (type_may_be_null(flag)) regs[regno].id = ++env->id_gen; return 0; case PTR_TO_MEM: mark_reg_known_zero(env, regs, regno); regs[regno].type = PTR_TO_MEM | flag; regs[regno].mem_size = 0; return 0; default: verifier_bug(env, "unexpected reg_type %d in %s\n", reg_type, __func__); return -EFAULT; }}
#define DEF_NOT_SUBREG (0)static void init_reg_state(struct bpf_verifier_env *env, struct bpf_func_state *state){ struct bpf_reg_state *regs = state->regs; int i;
for (i = 0; i < MAX_BPF_REG; i++) { bpf_mark_reg_not_init(env, ®s[i]); regs[i].subreg_def = DEF_NOT_SUBREG; }
/* frame pointer */ regs[BPF_REG_FP].type = PTR_TO_STACK; mark_reg_known_zero(env, regs, BPF_REG_FP); regs[BPF_REG_FP].frameno = state->frameno;}
static struct bpf_retval_range retval_range(s32 minval, s32 maxval){ /* * return_32bit is set to false by default and set explicitly * by the caller when necessary. */ return (struct bpf_retval_range){ minval, maxval, false };}
static void init_func_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int callsite, int frameno, int subprogno){ state->callsite = callsite; state->frameno = frameno; state->subprogno = subprogno; state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); mark_verifier_state_scratched(env);}
/* Similar to push_stack(), but for async callbacks */static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx, int subprog, bool is_sleepable){ struct bpf_verifier_stack_elem *elem; struct bpf_func_state *frame;
elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT); if (!elem) return ERR_PTR(-ENOMEM);
elem->insn_idx = insn_idx; elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; env->head = elem; env->stack_size++; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { verbose(env, "The sequence of %d jumps is too complex for async cb.\n", env->stack_size); return ERR_PTR(-E2BIG); } /* Unlike push_stack() do not bpf_copy_verifier_state(). * The caller state doesn't matter. * This is async callback. It starts in a fresh stack. * Initialize it similar to do_check_common(). */ elem->st.branches = 1; elem->st.in_sleepable = is_sleepable; frame = kzalloc_obj(*frame, GFP_KERNEL_ACCOUNT); if (!frame) return ERR_PTR(-ENOMEM); init_func_state(env, frame, BPF_MAIN_FUNC /* callsite */, 0 /* frameno within this callchain */, subprog /* subprog number within this prog */); elem->st.frame[0] = frame; return &elem->st;}
static int cmp_subprogs(const void *a, const void *b){ return ((struct bpf_subprog_info *)a)->start - ((struct bpf_subprog_info *)b)->start;}
/* Find subprogram that contains instruction at 'off' */struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off){ struct bpf_subprog_info *vals = env->subprog_info; int l, r, m;
if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0) return NULL;
l = 0; r = env->subprog_cnt - 1; while (l < r) { m = l + (r - l + 1) / 2; if (vals[m].start <= off) l = m; else r = m - 1; } return &vals[l];}
/* Find subprogram that starts exactly at 'off' */int bpf_find_subprog(struct bpf_verifier_env *env, int off){ struct bpf_subprog_info *p;
p = bpf_find_containing_subprog(env, off); if (!p || p->start != off) return -ENOENT; return p - env->subprog_info;}
static int add_subprog(struct bpf_verifier_env *env, int off){ int insn_cnt = env->prog->len; int ret;
if (off >= insn_cnt || off < 0) { verbose(env, "call to invalid destination\n"); return -EINVAL; } ret = bpf_find_subprog(env, off); if (ret >= 0) return ret; if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { verbose(env, "too many subprograms\n"); return -E2BIG; } /* determine subprog starts. The end is one before the next starts */ env->subprog_info[env->subprog_cnt++].start = off; sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL); return env->subprog_cnt - 1;}
static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env){ struct bpf_prog_aux *aux = env->prog->aux; struct btf *btf = aux->btf; const struct btf_type *t; u32 main_btf_id, id; const char *name; int ret, i;
/* Non-zero func_info_cnt implies valid btf */ if (!aux->func_info_cnt) return 0; main_btf_id = aux->func_info[0].type_id;
t = btf_type_by_id(btf, main_btf_id); if (!t) { verbose(env, "invalid btf id for main subprog in func_info\n"); return -EINVAL; }
name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:"); if (IS_ERR(name)) { ret = PTR_ERR(name); /* If there is no tag present, there is no exception callback */ if (ret == -ENOENT) ret = 0; else if (ret == -EEXIST) verbose(env, "multiple exception callback tags for main subprog\n"); return ret; }
ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC); if (ret < 0) { verbose(env, "exception callback '%s' could not be found in BTF\n", name); return ret; } id = ret; t = btf_type_by_id(btf, id); if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) { verbose(env, "exception callback '%s' must have global linkage\n", name); return -EINVAL; } ret = 0; for (i = 0; i < aux->func_info_cnt; i++) { if (aux->func_info[i].type_id != id) continue; ret = aux->func_info[i].insn_off; /* Further func_info and subprog checks will also happen * later, so assume this is the right insn_off for now. */ if (!ret) { verbose(env, "invalid exception callback insn_off in func_info: 0\n"); ret = -EINVAL; } } if (!ret) { verbose(env, "exception callback type id not found in func_info\n"); ret = -EINVAL; } return ret;}
#define MAX_KFUNC_BTFS 256
struct bpf_kfunc_btf { struct btf *btf; struct module *module; u16 offset;};
struct bpf_kfunc_btf_tab { struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; u32 nr_descs;};
static int kfunc_desc_cmp_by_id_off(const void *a, const void *b){ const struct bpf_kfunc_desc *d0 = a; const struct bpf_kfunc_desc *d1 = b;
/* func_id is not greater than BTF_MAX_TYPE */ return d0->func_id - d1->func_id ?: d0->offset - d1->offset;}
static int kfunc_btf_cmp_by_off(const void *a, const void *b){ const struct bpf_kfunc_btf *d0 = a; const struct bpf_kfunc_btf *d1 = b;
return d0->offset - d1->offset;}
static struct bpf_kfunc_desc *find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset){ struct bpf_kfunc_desc desc = { .func_id = func_id, .offset = offset, }; struct bpf_kfunc_desc_tab *tab;
tab = prog->aux->kfunc_tab; return bsearch(&desc, tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);}
int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 btf_fd_idx, u8 **func_addr){ const struct bpf_kfunc_desc *desc;
desc = find_kfunc_desc(prog, func_id, btf_fd_idx); if (!desc) return -EFAULT;
*func_addr = (u8 *)desc->addr; return 0;}
static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset){ struct bpf_kfunc_btf kf_btf = { .offset = offset }; struct bpf_kfunc_btf_tab *tab; struct bpf_kfunc_btf *b; struct module *mod; struct btf *btf; int btf_fd;
tab = env->prog->aux->kfunc_btf_tab; b = bsearch(&kf_btf, tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); if (!b) { if (tab->nr_descs == MAX_KFUNC_BTFS) { verbose(env, "too many different module BTFs\n"); return ERR_PTR(-E2BIG); }
if (bpfptr_is_null(env->fd_array)) { verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); return ERR_PTR(-EPROTO); }
if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, offset * sizeof(btf_fd), sizeof(btf_fd))) return ERR_PTR(-EFAULT);
btf = btf_get_by_fd(btf_fd); if (IS_ERR(btf)) { verbose(env, "invalid module BTF fd specified\n"); return btf; }
if (!btf_is_module(btf)) { verbose(env, "BTF fd for kfunc is not a module BTF\n"); btf_put(btf); return ERR_PTR(-EINVAL); }
mod = btf_try_get_module(btf); if (!mod) { btf_put(btf); return ERR_PTR(-ENXIO); }
b = &tab->descs[tab->nr_descs++]; b->btf = btf; b->module = mod; b->offset = offset;
/* sort() reorders entries by value, so b may no longer point * to the right entry after this */ sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_btf_cmp_by_off, NULL); } else { btf = b->btf; }
return btf;}
void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab){ if (!tab) return;
while (tab->nr_descs--) { module_put(tab->descs[tab->nr_descs].module); btf_put(tab->descs[tab->nr_descs].btf); } kfree(tab);}
static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset){ if (offset) { if (offset < 0) { /* In the future, this can be allowed to increase limit * of fd index into fd_array, interpreted as u16. */ verbose(env, "negative offset disallowed for kernel module function call\n"); return ERR_PTR(-EINVAL); }
return __find_kfunc_desc_btf(env, offset); } return btf_vmlinux ?: ERR_PTR(-ENOENT);}
#define KF_IMPL_SUFFIX "_impl"
static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, struct btf *btf, const char *func_name){ const struct btf_type *func; char buf[KSYM_NAME_LEN]; s32 impl_id; int len;
len = snprintf(buf, sizeof(buf), "%s%s", func_name, KF_IMPL_SUFFIX); if (len < 0 || len >= sizeof(buf)) { bpf_log(log, "function name %s%s is too long\n", func_name, KF_IMPL_SUFFIX); return NULL; }
impl_id = btf_find_by_name_kind(btf, buf, BTF_KIND_FUNC); if (impl_id <= 0) { bpf_log(log, "cannot find function %s in BTF\n", buf); return NULL; }
func = btf_type_by_id(btf, impl_id);
return btf_type_by_id(btf, func->type);}
static int fetch_kfunc_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, struct bpf_kfunc_meta *kfunc){ const struct btf_type *func, *func_proto; const char *func_name; u32 *kfunc_flags; struct btf *btf;
if (func_id <= 0) { verbose(env, "invalid kernel function btf_id %d\n", func_id); return -EINVAL; }
btf = find_kfunc_desc_btf(env, offset); if (IS_ERR(btf)) { verbose(env, "failed to find BTF for kernel function\n"); return PTR_ERR(btf); }
/* * Note that kfunc_flags may be NULL at this point, which * means that we couldn't find func_id in any relevant * kfunc_id_set. This most likely indicates an invalid kfunc * call. However we don't fail with an error here, * and let the caller decide what to do with NULL kfunc->flags. */ kfunc_flags = btf_kfunc_flags(btf, func_id, env->prog);
func = btf_type_by_id(btf, func_id); if (!func || !btf_type_is_func(func)) { verbose(env, "kernel btf_id %d is not a function\n", func_id); return -EINVAL; }
func_name = btf_name_by_offset(btf, func->name_off);
/* * An actual prototype of a kfunc with KF_IMPLICIT_ARGS flag * can be found through the counterpart _impl kfunc. */ if (kfunc_flags && (*kfunc_flags & KF_IMPLICIT_ARGS)) func_proto = find_kfunc_impl_proto(&env->log, btf, func_name); else func_proto = btf_type_by_id(btf, func->type);
if (!func_proto || !btf_type_is_func_proto(func_proto)) { verbose(env, "kernel function btf_id %d does not have a valid func_proto\n", func_id); return -EINVAL; }
memset(kfunc, 0, sizeof(*kfunc)); kfunc->btf = btf; kfunc->id = func_id; kfunc->name = func_name; kfunc->proto = func_proto; kfunc->flags = kfunc_flags;
return 0;}
int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset){ struct bpf_kfunc_btf_tab *btf_tab; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; struct bpf_prog_aux *prog_aux; struct bpf_kfunc_meta kfunc; struct bpf_kfunc_desc *desc; unsigned long addr; int err;
prog_aux = env->prog->aux; tab = prog_aux->kfunc_tab; btf_tab = prog_aux->kfunc_btf_tab; if (!tab) { if (!btf_vmlinux) { verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); return -ENOTSUPP; }
if (!env->prog->jit_requested) { verbose(env, "JIT is required for calling kernel function\n"); return -ENOTSUPP; }
if (!bpf_jit_supports_kfunc_call()) { verbose(env, "JIT does not support calling kernel function\n"); return -ENOTSUPP; }
if (!env->prog->gpl_compatible) { verbose(env, "cannot call kernel function from non-GPL compatible program\n"); return -EINVAL; }
tab = kzalloc_obj(*tab, GFP_KERNEL_ACCOUNT); if (!tab) return -ENOMEM; prog_aux->kfunc_tab = tab; }
/* func_id == 0 is always invalid, but instead of returning an error, be * conservative and wait until the code elimination pass before returning * error, so that invalid calls that get pruned out can be in BPF programs * loaded from userspace. It is also required that offset be untouched * for such calls. */ if (!func_id && !offset) return 0;
if (!btf_tab && offset) { btf_tab = kzalloc_obj(*btf_tab, GFP_KERNEL_ACCOUNT); if (!btf_tab) return -ENOMEM; prog_aux->kfunc_btf_tab = btf_tab; }
if (find_kfunc_desc(env->prog, func_id, offset)) return 0;
if (tab->nr_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; }
err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) return err;
addr = kallsyms_lookup_name(kfunc.name); if (!addr) { verbose(env, "cannot find address for kernel function %s\n", kfunc.name); return -EINVAL; }
if (bpf_dev_bound_kfunc_id(func_id)) { err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); if (err) return err; }
err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err;
desc = &tab->descs[tab->nr_descs++]; desc->func_id = func_id; desc->offset = offset; desc->addr = addr; desc->func_model = func_model; sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0;}
bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog){ return !!prog->aux->kfunc_tab;}
static int add_subprog_and_kfunc(struct bpf_verifier_env *env){ struct bpf_subprog_info *subprog = env->subprog_info; int i, ret, insn_cnt = env->prog->len, ex_cb_insn; struct bpf_insn *insn = env->prog->insnsi;
/* Add entry function. */ ret = add_subprog(env, 0); if (ret) return ret;
for (i = 0; i < insn_cnt; i++, insn++) { if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && !bpf_pseudo_kfunc_call(insn)) continue;
if (!env->bpf_capable) { verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); return -EPERM; }
if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) ret = add_subprog(env, i + insn->imm + 1); else ret = bpf_add_kfunc_call(env, insn->imm, insn->off);
if (ret < 0) return ret; }
ret = bpf_find_exception_callback_insn_off(env); if (ret < 0) return ret; ex_cb_insn = ret;
/* If ex_cb_insn > 0, this means that the main program has a subprog * marked using BTF decl tag to serve as the exception callback. */ if (ex_cb_insn) { ret = add_subprog(env, ex_cb_insn); if (ret < 0) return ret; for (i = 1; i < env->subprog_cnt; i++) { if (env->subprog_info[i].start != ex_cb_insn) continue; env->exception_callback_subprog = i; bpf_mark_subprog_exc_cb(env, i); break; } }
/* Add a fake 'exit' subprog which could simplify subprog iteration * logic. 'subprog_cnt' should not be increased. */ subprog[env->subprog_cnt].start = insn_cnt;
if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < env->subprog_cnt; i++) verbose(env, "func#%d @%d\n", i, subprog[i].start);
return 0;}
static int check_subprogs(struct bpf_verifier_env *env){ int i, subprog_start, subprog_end, off, cur_subprog = 0; struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len;
/* now check that all jumps are within the same subprog */ subprog_start = subprog[cur_subprog].start; subprog_end = subprog[cur_subprog + 1].start; for (i = 0; i < insn_cnt; i++) { u8 code = insn[i].code;
if (code == (BPF_JMP | BPF_CALL) && insn[i].src_reg == 0 && insn[i].imm == BPF_FUNC_tail_call) { subprog[cur_subprog].has_tail_call = true; subprog[cur_subprog].tail_call_reachable = true; } if (BPF_CLASS(code) == BPF_LD && (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) subprog[cur_subprog].has_ld_abs = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) goto next; if (BPF_OP(code) == BPF_EXIT) { subprog[cur_subprog].exit_idx = i; goto next; } off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); return -EINVAL; }next: if (i == subprog_end - 1) { /* to avoid fall-through from one subprog into another * the last insn of the subprog should be either exit * or unconditional jump back or bpf_throw call */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && code != (BPF_JMP | BPF_JA)) { verbose(env, "last insn is not an exit or jmp\n"); return -EINVAL; } subprog_start = subprog_end; cur_subprog++; if (cur_subprog < env->subprog_cnt) subprog_end = subprog[cur_subprog + 1].start; } } return 0;}
/* * Sort subprogs in topological order so that leaf subprogs come first and * their callers come later. This is a DFS post-order traversal of the call * graph. Scan only reachable instructions (those in the computed postorder) of * the current subprog to discover callees (direct subprogs and sync * callbacks). */static int sort_subprogs_topo(struct bpf_verifier_env *env){ struct bpf_subprog_info *si = env->subprog_info; int *insn_postorder = env->cfg.insn_postorder; struct bpf_insn *insn = env->prog->insnsi; int cnt = env->subprog_cnt; int *dfs_stack = NULL; int top = 0, order = 0; int i, ret = 0; u8 *color = NULL;
color = kvzalloc_objs(*color, cnt, GFP_KERNEL_ACCOUNT); dfs_stack = kvmalloc_objs(*dfs_stack, cnt, GFP_KERNEL_ACCOUNT); if (!color || !dfs_stack) { ret = -ENOMEM; goto out; }
/* * DFS post-order traversal. * Color values: 0 = unvisited, 1 = on stack, 2 = done. */ for (i = 0; i < cnt; i++) { if (color[i]) continue; color[i] = 1; dfs_stack[top++] = i;
while (top > 0) { int cur = dfs_stack[top - 1]; int po_start = si[cur].postorder_start; int po_end = si[cur + 1].postorder_start; bool pushed = false; int j;
for (j = po_start; j < po_end; j++) { int idx = insn_postorder[j]; int callee;
if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx])) continue; callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); if (callee < 0) { ret = -EFAULT; goto out; } if (color[callee] == 2) continue; if (color[callee] == 1) { if (bpf_pseudo_func(&insn[idx])) continue; verbose(env, "recursive call from %s() to %s()\n", subprog_name(env, cur), subprog_name(env, callee)); ret = -EINVAL; goto out; } color[callee] = 1; dfs_stack[top++] = callee; pushed = true; break; }
if (!pushed) { color[cur] = 2; env->subprog_topo_order[order++] = cur; top--; } } }
if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < cnt; i++) verbose(env, "topo_order[%d] = %s\n", i, subprog_name(env, env->subprog_topo_order[i]));out: kvfree(dfs_stack); kvfree(color); return ret;}
static void mark_stack_slots_scratched(struct bpf_verifier_env *env, int spi, int nr_slots){ int i;
for (i = 0; i < nr_slots; i++) mark_stack_slot_scratched(env, spi - i);}
/* This function is supposed to be used by the following 32-bit optimization * code only. It returns TRUE if the source or destination register operates * on 64-bit, otherwise return FALSE. */bool bpf_is_reg64(struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t){ u8 code, class, op;
code = insn->code; class = BPF_CLASS(code); op = BPF_OP(code); if (class == BPF_JMP) { /* BPF_EXIT for "main" will reach here. Return TRUE * conservatively. */ if (op == BPF_EXIT) return true; if (op == BPF_CALL) { /* BPF to BPF call will reach here because of marking * caller saved clobber with DST_OP_NO_MARK for which we * don't care the register def because they are anyway * marked as NOT_INIT already. */ if (insn->src_reg == BPF_PSEUDO_CALL) return false; /* Helper call will reach here because of arg type * check, conservatively return TRUE. */ if (t == SRC_OP) return true;
return false; } }
if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) return false;
if (class == BPF_ALU64 || class == BPF_JMP || (class == BPF_ALU && op == BPF_END && insn->imm == 64)) return true;
if (class == BPF_ALU || class == BPF_JMP32) return false;
if (class == BPF_LDX) { if (t != SRC_OP) return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; /* LDX source must be ptr. */ return true; }
if (class == BPF_STX) { /* BPF_STX (including atomic variants) has one or more source * operands, one of which is a ptr. Check whether the caller is * asking about it. */ if (t == SRC_OP && reg->type != SCALAR_VALUE) return true; return BPF_SIZE(code) == BPF_DW; }
if (class == BPF_LD) { u8 mode = BPF_MODE(code);
/* LD_IMM64 */ if (mode == BPF_IMM) return true;
/* Both LD_IND and LD_ABS return 32-bit data. */ if (t != SRC_OP) return false;
/* Implicit ctx ptr. */ if (regno == BPF_REG_6) return true;
/* Explicit source could be any width. */ return true; }
if (class == BPF_ST) /* The only source register for BPF_ST is a ptr. */ return true;
/* Conservatively return true at default. */ return true;}
static void mark_insn_zext(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ s32 def_idx = reg->subreg_def;
if (def_idx == DEF_NOT_SUBREG) return;
env->insn_aux_data[def_idx - 1].zext_dst = true; /* The dst will be zero extended, so won't be sub-register anymore. */ reg->subreg_def = DEF_NOT_SUBREG;}
static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, enum bpf_reg_arg_type t){ struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; struct bpf_reg_state *reg; bool rw64;
mark_reg_scratched(env, regno);
reg = ®s[regno]; rw64 = bpf_is_reg64(insn, regno, reg, t); if (t == SRC_OP) { /* check whether register used as source operand can be read */ if (reg->type == NOT_INIT) { verbose(env, "R%d !read_ok\n", regno); return -EACCES; } /* We don't need to worry about FP liveness because it's read-only */ if (regno == BPF_REG_FP) return 0;
if (rw64) mark_insn_zext(env, reg);
return 0; } else { /* check whether register used as dest operand can be written to */ if (regno == BPF_REG_FP) { verbose(env, "frame pointer is read only\n"); return -EACCES; } reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; if (t == DST_OP) mark_reg_unknown(env, regs, regno); } return 0;}
static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, enum bpf_reg_arg_type t){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe];
return __check_reg_arg(env, state->regs, regno, t);}
static void mark_indirect_target(struct bpf_verifier_env *env, int idx){ env->insn_aux_data[idx].indirect_target = true;}
#define LR_FRAMENO_BITS 4#define LR_SPI_BITS 6#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)#define LR_SIZE_BITS 4#define LR_FRAMENO_MASK ((1ull << LR_FRAMENO_BITS) - 1)#define LR_SPI_MASK ((1ull << LR_SPI_BITS) - 1)#define LR_SIZE_MASK ((1ull << LR_SIZE_BITS) - 1)#define LR_SPI_OFF LR_FRAMENO_BITS#define LR_IS_REG_OFF (LR_SPI_BITS + LR_FRAMENO_BITS)#define LINKED_REGS_MAX 5
static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS));static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS));static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64);
struct linked_reg { u8 frameno; union { u8 spi; u8 regno; }; bool is_reg;};
struct linked_regs { int cnt; struct linked_reg entries[LINKED_REGS_MAX];};
static struct linked_reg *linked_regs_push(struct linked_regs *s){ if (s->cnt < LINKED_REGS_MAX) return &s->entries[s->cnt++];
return NULL;}
/* * Use u64 as a vector of 5 11-bit values, use first 4-bits to track * number of elements currently in stack. * Pack one history entry for linked registers as 11 bits in the following format: * - 4-bits frameno * - 6-bits spi_or_reg * - 1-bit is_reg */static u64 linked_regs_pack(struct linked_regs *s){ u64 val = 0; int i;
for (i = 0; i < s->cnt; ++i) { struct linked_reg *e = &s->entries[i]; u64 tmp = 0;
tmp |= e->frameno; tmp |= e->spi << LR_SPI_OFF; tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF;
val <<= LR_ENTRY_BITS; val |= tmp; } val <<= LR_SIZE_BITS; val |= s->cnt; return val;}
static void linked_regs_unpack(u64 val, struct linked_regs *s){ int i;
s->cnt = val & LR_SIZE_MASK; val >>= LR_SIZE_BITS;
for (i = 0; i < s->cnt; ++i) { struct linked_reg *e = &s->entries[i];
e->frameno = val & LR_FRAMENO_MASK; e->spi = (val >> LR_SPI_OFF) & LR_SPI_MASK; e->is_reg = (val >> LR_IS_REG_OFF) & 0x1; val >>= LR_ENTRY_BITS; }}
static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn){ const struct btf_type *func; struct btf *desc_btf;
if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) return NULL;
desc_btf = find_kfunc_desc_btf(data, insn->off); if (IS_ERR(desc_btf)) return "<error>";
func = btf_type_by_id(desc_btf, insn->imm); return btf_name_by_offset(desc_btf, func->name_off);}
void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn){ const struct bpf_insn_cbs cbs = { .cb_call = disasm_kfunc_name, .cb_print = verbose, .private_data = env, };
print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);}
/* If any register R in hist->linked_regs is marked as precise in bt, * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs. */void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist){ struct linked_regs linked_regs; bool some_precise = false; int i;
if (!hist || hist->linked_regs == 0) return;
linked_regs_unpack(hist->linked_regs, &linked_regs); for (i = 0; i < linked_regs.cnt; ++i) { struct linked_reg *e = &linked_regs.entries[i];
if ((e->is_reg && bt_is_frame_reg_set(bt, e->frameno, e->regno)) || (!e->is_reg && bt_is_frame_slot_set(bt, e->frameno, e->spi))) { some_precise = true; break; } }
if (!some_precise) return;
for (i = 0; i < linked_regs.cnt; ++i) { struct linked_reg *e = &linked_regs.entries[i];
if (e->is_reg) bpf_bt_set_frame_reg(bt, e->frameno, e->regno); else bpf_bt_set_frame_slot(bt, e->frameno, e->spi); }}
int mark_chain_precision(struct bpf_verifier_env *env, int regno){ return bpf_mark_chain_precision(env, env->cur_state, regno, NULL);}
/* mark_chain_precision_batch() assumes that env->bt is set in the caller to * desired reg and stack masks across all relevant frames */static int mark_chain_precision_batch(struct bpf_verifier_env *env, struct bpf_verifier_state *starting_state){ return bpf_mark_chain_precision(env, starting_state, -1, NULL);}
static bool is_spillable_regtype(enum bpf_reg_type type){ switch (base_type(type)) { case PTR_TO_MAP_VALUE: case PTR_TO_STACK: case PTR_TO_CTX: case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_PACKET_END: case PTR_TO_FLOW_KEYS: case CONST_PTR_TO_MAP: case PTR_TO_SOCKET: case PTR_TO_SOCK_COMMON: case PTR_TO_TCP_SOCK: case PTR_TO_XDP_SOCK: case PTR_TO_BTF_ID: case PTR_TO_BUF: case PTR_TO_MEM: case PTR_TO_FUNC: case PTR_TO_MAP_KEY: case PTR_TO_ARENA: return true; default: return false; }}
/* check if register is a constant scalar value */static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32){ return reg->type == SCALAR_VALUE && tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off);}
/* assuming is_reg_const() is true, return constant value of a register */static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32){ return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;}
static bool __is_pointer_value(bool allow_ptr_leaks, const struct bpf_reg_state *reg){ if (allow_ptr_leaks) return false;
return reg->type != SCALAR_VALUE;}
static void clear_scalar_id(struct bpf_reg_state *reg){ reg->id = 0; reg->delta = 0;}
static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, struct bpf_reg_state *src_reg){ if (src_reg->type != SCALAR_VALUE) return; /* * The verifier is processing rX = rY insn and * rY->id has special linked register already. * Cleared it, since multiple rX += const are not supported. */ if (src_reg->id & BPF_ADD_CONST) clear_scalar_id(src_reg); /* * Ensure that src_reg has a valid ID that will be copied to * dst_reg and then will be used by sync_linked_regs() to * propagate min/max range. */ if (!src_reg->id && !tnum_is_const(src_reg->var_off)) src_reg->id = ++env->id_gen;}
static void save_register_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi, struct bpf_reg_state *reg, int size){ int i;
state->stack[spi].spilled_ptr = *reg;
for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) state->stack[spi].slot_type[i - 1] = STACK_SPILL;
/* size < 8 bytes spill */ for (; i; i--) mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);}
static bool is_bpf_st_mem(struct bpf_insn *insn){ return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;}
static int get_reg_width(struct bpf_reg_state *reg){ return fls64(reg_umax(reg));}
/* See comment for mark_fastcall_pattern_for_call() */static void check_fastcall_stack_contract(struct bpf_verifier_env *env, struct bpf_func_state *state, int insn_idx, int off){ struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; struct bpf_insn_aux_data *aux = env->insn_aux_data; int i;
if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern) return; /* access to the region [max_stack_depth .. fastcall_stack_off) * from something that is not a part of the fastcall pattern, * disable fastcall rewrites for current subprogram by setting * fastcall_stack_off to a value smaller than any possible offset. */ subprog->fastcall_stack_off = S16_MIN; /* reset fastcall aux flags within subprogram, * happens at most once per subprogram */ for (i = subprog->start; i < (subprog + 1)->start; ++i) { aux[i].fastcall_spills_num = 0; aux[i].fastcall_pattern = 0; }}
static void scrub_special_slot(struct bpf_func_state *state, int spi){ int i;
/* regular write of data into stack destroys any spilled ptr */ state->stack[spi].spilled_ptr.type = NOT_INIT; /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ if (is_stack_slot_special(&state->stack[spi])) for (i = 0; i < BPF_REG_SIZE; i++) scrub_spilled_slot(&state->stack[spi].slot_type[i]);}
/* check_stack_{read,write}_fixed_off functions track spill/fill of registers, * stack boundary and alignment are checked in check_mem_access() */static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* stack frame we're writing to */ struct bpf_func_state *state, int off, int size, int value_regno, int insn_idx){ struct bpf_func_state *cur; /* state of the current function */ int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_reg_state *reg = NULL; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = state->frameno;
/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && bpf_is_spilled_reg(&state->stack[spi]) && !bpf_is_spilled_scalar_reg(&state->stack[spi]) && size != BPF_REG_SIZE) { verbose(env, "attempt to corrupt spilled pointer on stack\n"); return -EACCES; }
cur = env->cur_state->frame[env->cur_state->curframe]; if (value_regno >= 0) reg = &cur->regs[value_regno]; if (!env->bypass_spec_v4) { bool sanitize = reg && is_spillable_regtype(reg->type);
for (i = 0; i < size; i++) { u8 type = state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE];
if (type != STACK_MISC && type != STACK_ZERO) { sanitize = true; break; } }
if (sanitize) env->insn_aux_data[insn_idx].nospec_result = true; }
err = destroy_if_dynptr_stack_slot(env, state, spi); if (err) return err;
check_fastcall_stack_contract(env, state, insn_idx, off); mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { bool reg_value_fits;
reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size; /* Make sure that reg had an ID to build a relation on spill. */ if (reg_value_fits) assign_scalar_id_before_mov(env, reg); save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) state->stack[spi].spilled_ptr.id = 0; } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0];
memset(tmp_reg, 0, sizeof(*tmp_reg)); __mark_reg_known(tmp_reg, insn->imm); tmp_reg->type = SCALAR_VALUE; save_register_state(env, state, spi, tmp_reg, size); } else if (reg && is_spillable_regtype(reg->type)) { /* register containing pointer is being spilled into stack */ if (size != BPF_REG_SIZE) { verbose_linfo(env, insn_idx, "; "); verbose(env, "invalid size of register spill\n"); return -EACCES; } if (state != cur && reg->type == PTR_TO_STACK) { verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); return -EINVAL; } save_register_state(env, state, spi, reg, size); } else { u8 type = STACK_MISC;
scrub_special_slot(state, spi);
/* when we zero initialize stack slots mark them as such */ if ((reg && bpf_register_is_null(reg)) || (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) { /* STACK_ZERO case happened because register spill * wasn't properly aligned at the stack slot boundary, * so it's not a register spill anymore; force * originating register to be precise to make * STACK_ZERO correct for subsequent states */ err = mark_chain_precision(env, value_regno); if (err) return err; type = STACK_ZERO; }
/* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; insn_flags = 0; /* not a register spill */ }
if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, hist_spi, hist_frame, 0); return 0;}
/* Write the stack: 'stack[ptr_reg + off] = value_regno'. 'ptr_reg' is * known to contain a variable offset. * This function checks whether the write is permitted and conservatively * tracks the effects of the write, considering that each stack slot in the * dynamic range is potentially written to. * * 'value_regno' can be -1, meaning that an unknown value is being written to * the stack. * * Spilled pointers in range are not marked as written because we don't know * what's going to be actually written. This means that read propagation for * future reads cannot be terminated by this write. * * For privileged programs, uninitialized stack slots are considered * initialized by this write (even though we don't know exactly what offsets * are going to be written to). The idea is that we don't want the verifier to * reject future reads that access slots written to through variable offsets. */static int check_stack_write_var_off(struct bpf_verifier_env *env, /* func where register points to */ struct bpf_func_state *state, struct bpf_reg_state *ptr_reg, int off, int size, int value_regno, int insn_idx){ struct bpf_func_state *cur; /* state of the current function */ int min_off, max_off; int i, err; struct bpf_reg_state *value_reg = NULL; struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; bool writing_zero = false; /* set if the fact that we're writing a zero is used to let any * stack slots remain STACK_ZERO */ bool zero_used = false;
cur = env->cur_state->frame[env->cur_state->curframe]; min_off = reg_smin(ptr_reg) + off; max_off = reg_smax(ptr_reg) + off + size; if (value_regno >= 0) value_reg = &cur->regs[value_regno]; if ((value_reg && bpf_register_is_null(value_reg)) || (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) writing_zero = true;
for (i = min_off; i < max_off; i++) { int spi;
spi = bpf_get_spi(i); err = destroy_if_dynptr_stack_slot(env, state, spi); if (err) return err; }
check_fastcall_stack_contract(env, state, insn_idx, min_off); /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { u8 new_type, *stype; int slot, spi;
slot = -i - 1; spi = slot / BPF_REG_SIZE; stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; mark_stack_slot_scratched(env, spi);
if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { /* Reject the write if range we may write to has not * been initialized beforehand. If we didn't reject * here, the ptr status would be erased below (even * though not all slots are actually overwritten), * possibly opening the door to leaks. * * We do however catch STACK_INVALID case below, and * only allow reading possibly uninitialized memory * later for CAP_PERFMON, as the write may not happen to * that slot. */ verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", insn_idx, i); return -EINVAL; }
/* If writing_zero and the spi slot contains a spill of value 0, * maintain the spill type. */ if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(&state->stack[spi])) { struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;
if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { zero_used = true; continue; } }
/* * Scrub slots if variable-offset stack write goes over spilled pointers. * Otherwise bpf_is_spilled_reg() may == true && spilled_ptr.type == NOT_INIT * and valid program is rejected by check_stack_read_fixed_off() * with obscure "invalid size of register fill" message. */ scrub_special_slot(state, spi);
/* Update the slot type. */ new_type = STACK_MISC; if (writing_zero && *stype == STACK_ZERO) { new_type = STACK_ZERO; zero_used = true; } /* If the slot is STACK_INVALID, we check whether it's OK to * pretend that it will be initialized by this write. The slot * might not actually be written to, and so if we mark it as * initialized future reads might leak uninitialized memory. * For privileged programs, we will accept such reads to slots * that may or may not be written because, if we're reject * them, the error would be too confusing. * Conservatively, treat STACK_POISON in a similar way. */ if ((*stype == STACK_INVALID || *stype == STACK_POISON) && !env->allow_uninit_stack) { verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", insn_idx, i); return -EINVAL; } *stype = new_type; } if (zero_used) { /* backtracking doesn't work for STACK_ZERO yet. */ err = mark_chain_precision(env, value_regno); if (err) return err; } return 0;}
/* When register 'dst_regno' is assigned some values from stack[min_off, * max_off), we set the register's type according to the types of the * respective stack slots. If all the stack values are known to be zeros, then * so is the destination reg. Otherwise, the register is considered to be * SCALAR. This function does not deal with register filling; the caller must * ensure that all spilled registers in the stack range have been marked as * read. */static void mark_reg_stack_read(struct bpf_verifier_env *env, /* func where src register points to */ struct bpf_func_state *ptr_state, int min_off, int max_off, int dst_regno){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; int i, slot, spi; u8 *stype; int zeros = 0;
for (i = min_off; i < max_off; i++) { slot = -i - 1; spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); stype = ptr_state->stack[spi].slot_type; if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) break; zeros++; } if (zeros == max_off - min_off) { /* Any access_size read into register is zero extended, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); } else { /* have read misc data from the stack */ mark_reg_unknown(env, state->regs, dst_regno); }}
/* Read the stack at 'off' and put the results into the register indicated by * 'dst_regno'. It handles reg filling if the addressed stack slot is a * spilled reg. * * 'dst_regno' can be -1, meaning that the read value is not going to a * register. * * The access is assumed to be within the current stack bounds. */static int check_stack_read_fixed_off(struct bpf_verifier_env *env, /* func where src register points to */ struct bpf_func_state *reg_state, int off, int size, int dst_regno){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno;
stype = reg_state->stack[spi].slot_type; reg = ®_state->stack[spi].spilled_ptr;
mark_stack_slot_scratched(env, spi); check_fastcall_stack_contract(env, state, env->insn_idx, off);
if (bpf_is_spilled_reg(®_state->stack[spi])) { u8 spill_size = 1;
for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) spill_size++;
if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { if (reg->type != SCALAR_VALUE) { verbose_linfo(env, env->insn_idx, "; "); verbose(env, "invalid size of register fill\n"); return -EACCES; }
if (dst_regno < 0) return 0;
if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { /* The earlier check_reg_arg() has decided the * subreg_def for this insn. Save it first. */ s32 subreg_def = state->regs[dst_regno].subreg_def;
if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation * with the destination register on fill. */ assign_scalar_id_before_mov(env, reg); state->regs[dst_regno] = *reg; state->regs[dst_regno].subreg_def = subreg_def;
/* Break the relation on a narrowing fill. * coerce_reg_to_size will adjust the boundaries. */ if (get_reg_width(reg) > size * BITS_PER_BYTE) clear_scalar_id(&state->regs[dst_regno]); } else { int spill_cnt = 0, zero_cnt = 0;
for (i = 0; i < size; i++) { type = stype[(slot - i) % BPF_REG_SIZE]; if (type == STACK_SPILL) { spill_cnt++; continue; } if (type == STACK_MISC) continue; if (type == STACK_ZERO) { zero_cnt++; continue; } if (type == STACK_INVALID && env->allow_uninit_stack) continue; if (type == STACK_POISON) { verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n", off, i, size); } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); } return -EACCES; }
if (spill_cnt == size && tnum_is_const(reg->var_off) && reg->var_off.value == 0) { __mark_reg_const_zero(env, &state->regs[dst_regno]); /* this IS register fill, so keep insn_flags */ } else if (zero_cnt == size) { /* similarly to mark_reg_stack_read(), preserve zeroes */ __mark_reg_const_zero(env, &state->regs[dst_regno]); insn_flags = 0; /* not restoring original register state */ } else { mark_reg_unknown(env, state->regs, dst_regno); insn_flags = 0; /* not restoring original register state */ } } } else if (dst_regno >= 0) { /* restore register state from stack */ if (env->bpf_capable) /* Ensure stack slot has an ID to build a relation * with the destination register on fill. */ assign_scalar_id_before_mov(env, reg); state->regs[dst_regno] = *reg; /* mark reg as written since spilled pointer state likely * has its liveness marks cleared by is_state_visited() * which resets stack/reg liveness for state transitions */ } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { /* If dst_regno==-1, the caller is asking us whether * it is acceptable to use this value as a SCALAR_VALUE * (e.g. for XADD). * We must not allow unprivileged callers to do that * with spilled pointers. */ verbose(env, "leaking pointer from stack off %d\n", off); return -EACCES; } } else { for (i = 0; i < size; i++) { type = stype[(slot - i) % BPF_REG_SIZE]; if (type == STACK_MISC) continue; if (type == STACK_ZERO) continue; if (type == STACK_INVALID && env->allow_uninit_stack) continue; if (type == STACK_POISON) { verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n", off, i, size); } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); } return -EACCES; } if (dst_regno >= 0) mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); insn_flags = 0; /* we are not restoring spilled register */ } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, hist_spi, hist_frame, 0); return 0;}
enum bpf_access_src { ACCESS_DIRECT = 1, /* the access is performed by an instruction */ ACCESS_HELPER = 2, /* the access is performed by a helper */};
static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int access_size, bool zero_size_allowed, enum bpf_access_type type, struct bpf_call_arg_meta *meta);
static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno){ return cur_regs(env) + regno;}
/* Read the stack at 'reg + off' and put the result into the register * 'dst_regno'. * 'off' includes the pointer register's fixed offset(i.e. 'reg->off'), * but not its variable offset. * 'size' is assumed to be <= reg size and the access is assumed to be aligned. * * As opposed to check_stack_read_fixed_off, this function doesn't deal with * filling registers (i.e. reads of spilled register cannot be detected when * the offset is not fixed). We conservatively mark 'dst_regno' as containing * SCALAR_VALUE. That's why we assert that the 'reg' has a variable * offset; for a fixed offset check_stack_read_fixed_off should be used * instead. */static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t ptr_argno, int off, int size, int dst_regno){ struct bpf_func_state *ptr_state = bpf_func(env, reg); int err; int min_off, max_off;
/* Note that we pass a NULL meta, so raw access will not be permitted. */ err = check_stack_range_initialized(env, reg, ptr_argno, off, size, false, BPF_READ, NULL); if (err) return err;
min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0;}
/* check_stack_read dispatches to check_stack_read_fixed_off or * check_stack_read_var_off. * * The caller must ensure that the offset falls within the allocated stack * bounds. * * 'dst_regno' is a register which will receive the value from the stack. It * can be -1, meaning that the read value is not going to a register. */static int check_stack_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t ptr_argno, int off, int size, int dst_regno){ struct bpf_func_state *state = bpf_func(env, reg); int err; /* Some accesses are only permitted with a static offset. */ bool var_off = !tnum_is_const(reg->var_off);
/* The offset is required to be static when reads don't go to a * register, in order to not leak pointers (see * check_stack_read_fixed_off). */ if (dst_regno < 0 && var_off) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity * since it requires corresponding support in Spectre masking for stack * ALU. See also retrieve_ptr_limit(). The check in * check_stack_access_for_ptr_arithmetic() called by * adjust_ptr_min_max_vals() prevents users from creating stack pointers * with variable offsets, therefore no check is required here. Further, * just checking it here would be insufficient as speculative stack * writes could still lead to unsafe speculative behaviour. */ if (!var_off) { off += reg->var_off.value; err = check_stack_read_fixed_off(env, state, off, size, dst_regno); } else { /* Variable offset stack reads need more conservative handling * than fixed offset ones. Note that dst_regno >= 0 on this * branch. */ err = check_stack_read_var_off(env, reg, ptr_argno, off, size, dst_regno); } return err;}
/* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * * 'reg' is the register used as a pointer into the stack. * 'value_regno' is the register whose value we're writing to the stack. It can * be -1, meaning that we're not writing from a register. * * The caller must ensure that the offset falls within the maximum stack size. */static int check_stack_write(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int size, int value_regno, int insn_idx){ struct bpf_func_state *state = bpf_func(env, reg); int err;
if (tnum_is_const(reg->var_off)) { off += reg->var_off.value; err = check_stack_write_fixed_off(env, state, off, size, value_regno, insn_idx); } else { /* Variable offset stack reads need more conservative handling * than fixed offset ones. */ err = check_stack_write_var_off(env, state, reg, off, size, value_regno, insn_idx); } return err;}
/* * Write a value to the outgoing stack arg area. * off is a negative offset from r11 (e.g. -8 for arg6, -16 for arg7). */static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_state *state, int off, struct bpf_reg_state *value_reg){ int max_stack_arg_regs = MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS; struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; int spi = -off / BPF_REG_SIZE - 1; struct bpf_reg_state *arg; int err;
if (spi >= max_stack_arg_regs) { verbose(env, "stack arg write offset %d exceeds max %d stack args\n", off, max_stack_arg_regs); return -EINVAL; }
err = grow_stack_arg_slots(env, state, spi + 1); if (err) return err;
/* Track the max outgoing stack arg slot count. */ if (spi + 1 > subprog->max_out_stack_arg_cnt) subprog->max_out_stack_arg_cnt = spi + 1;
if (value_reg) { state->stack_arg_regs[spi] = *value_reg; } else { /* BPF_ST: store immediate, treat as scalar */ arg = &state->stack_arg_regs[spi]; arg->type = SCALAR_VALUE; __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); } state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0);}
/* * Read a value from the incoming stack arg area. * off is a positive offset from r11 (e.g. +8 for arg6, +16 for arg7). */static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_state *state, int off, int dst_regno){ struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; struct bpf_verifier_state *vstate = env->cur_state; int spi = off / BPF_REG_SIZE - 1; struct bpf_func_state *caller, *cur; struct bpf_reg_state *arg;
if (state->no_stack_arg_load) { verbose(env, "r11 load must be before any r11 store or call insn\n"); return -EINVAL; }
if (spi + 1 > bpf_in_stack_arg_cnt(subprog)) { verbose(env, "invalid read from stack arg off %d depth %d\n", off, bpf_in_stack_arg_cnt(subprog) * BPF_REG_SIZE); return -EACCES; }
caller = vstate->frame[vstate->curframe - 1]; arg = &caller->stack_arg_regs[spi]; cur = vstate->frame[vstate->curframe]; cur->regs[dst_regno] = *arg; return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0);}
static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx){ struct bpf_func_state *caller = cur_func(env); int spi = arg_idx - MAX_BPF_FUNC_REG_ARGS;
bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi); return mark_chain_precision_batch(env, env->cur_state);}
static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, int nargs){ int i, spi;
for (i = MAX_BPF_FUNC_REG_ARGS; i < nargs; i++) { spi = i - MAX_BPF_FUNC_REG_ARGS; if (spi >= caller->out_stack_arg_cnt || caller->stack_arg_regs[spi].type == NOT_INIT) { verbose(env, "callee expects %d args, stack arg%d is not initialized\n", nargs, spi + 1); return -EFAULT; } }
return 0;}
static struct bpf_reg_state *get_func_arg_reg(struct bpf_func_state *caller, struct bpf_reg_state *regs, int arg){ if (arg < MAX_BPF_FUNC_REG_ARGS) return ®s[arg + 1];
return &caller->stack_arg_regs[arg - MAX_BPF_FUNC_REG_ARGS];}
static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int size, enum bpf_access_type type){ struct bpf_map *map = reg->map_ptr; u32 cap = bpf_map_flags_to_cap(map);
if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n", map->value_size, reg_smin(reg) + off, size); return -EACCES; }
if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n", map->value_size, reg_smin(reg) + off, size); return -EACCES; }
return 0;}
/* check read/write into memory region (e.g., map value, ringbuf sample, etc) */static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, u32 mem_size, bool zero_size_allowed){ bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
if (off >= 0 && size_ok && (u64)off + size <= mem_size) return 0;
switch (reg->type) { case PTR_TO_MAP_KEY: verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", mem_size, off, size); break; case PTR_TO_MAP_VALUE: verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", mem_size, off, size); break; case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_PACKET_END: verbose(env, "invalid access to packet, off=%d size=%d, %s(id=%d,off=%d,r=%d)\n", off, size, reg_arg_name(env, argno), reg->id, off, mem_size); break; case PTR_TO_CTX: verbose(env, "invalid access to context, ctx_size=%d off=%d size=%d\n", mem_size, off, size); break; case PTR_TO_MEM: default: verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", mem_size, off, size); }
return -EACCES;}
/* check read/write into a memory region with possible variable offset */static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, u32 mem_size, bool zero_size_allowed){ int err;
/* We may have adjusted the register pointing to memory region, so we * need to try adding each of min_value and max_value to off * to make sure our theoretical access will be safe. * * The minimum value is only important with signed * comparisons where we can't assume the floor of a * value is 0. If we are using signed variables for our * index'es we need to make sure that whatever we use * will have a set floor within our range. */ if (reg_smin(reg) < 0 && (reg_smin(reg) == S64_MIN || (off + reg_smin(reg) != (s64)(s32)(off + reg_smin(reg))) || reg_smin(reg) + off < 0)) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); return -EACCES; } err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s min value is outside of the allowed memory range\n", reg_arg_name(env, argno)); return err; }
/* If we haven't set a max value then we need to bail since we can't be * sure we won't do bad things. * If reg_umax(reg) + off could overflow, treat that as unbounded too. */ if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", reg_arg_name(env, argno)); return -EACCES; } err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s max value is outside of the allowed memory range\n", reg_arg_name(env, argno)); return err; }
return 0;}
static int __check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, bool fixed_off_ok){ /* Access to this pointer-typed register or passing it to a helper * is only allowed in its original, unmodified form. */
if (!tnum_is_const(reg->var_off)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable %s access var_off=%s disallowed\n", reg_type_str(env, reg->type), tn_buf); return -EACCES; }
if (reg_smin(reg) < 0) { verbose(env, "negative offset %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); return -EACCES; }
if (!fixed_off_ok && reg->var_off.value != 0) { verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); return -EACCES; }
return 0;}
static int check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno){ return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);}
static int map_kptr_match_type(struct bpf_verifier_env *env, struct btf_field *kptr_field, struct bpf_reg_state *reg, u32 regno){ const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); int perm_flags; const char *reg_name = "";
if (base_type(reg->type) != PTR_TO_BTF_ID) goto bad_type;
if (btf_is_kernel(reg->btf)) { perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
/* Only unreferenced case accepts untrusted pointers */ if (kptr_field->type == BPF_KPTR_UNREF) perm_flags |= PTR_UNTRUSTED; } else { perm_flags = PTR_MAYBE_NULL | MEM_ALLOC; if (kptr_field->type == BPF_KPTR_PERCPU) perm_flags |= MEM_PERCPU; }
if (type_flag(reg->type) & ~perm_flags) goto bad_type;
/* We need to verify reg->type and reg->btf, before accessing reg->btf */ reg_name = btf_type_name(reg->btf, reg->btf_id);
/* For ref_ptr case, release function check should ensure we get one * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the * normal store of unreferenced kptr, we must ensure var_off is zero. * Since ref_ptr cannot be accessed directly by BPF insns, check for * reg->id is not needed here. */ if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true)) return -EACCES;
/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and * we also need to take into account the reg->var_off. * * We want to support cases like: * * struct foo { * struct bar br; * struct baz bz; * }; * * struct foo *v; * v = func(); // PTR_TO_BTF_ID * val->foo = v; // reg->var_off is zero, btf and btf_id match type * val->bar = &v->br; // reg->var_off is still zero, but we need to retry with * // first member type of struct after comparison fails * val->baz = &v->bz; // reg->var_off is non-zero, so struct needs to be walked * // to match type * * In the kptr_ref case, check_func_arg_reg_off already ensures reg->var_off * is zero. We must also ensure that btf_struct_ids_match does not walk * the struct to match type against first member of struct, i.e. reject * second case from above. Hence, when type is BPF_KPTR_REF, we set * strict mode to true for type match. */ if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, kptr_field->kptr.btf, kptr_field->kptr.btf_id, kptr_field->type != BPF_KPTR_UNREF)) goto bad_type; return 0;bad_type: verbose(env, "invalid kptr access, R%d type=%s%s ", regno, reg_type_str(env, reg->type), reg_name); verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); if (kptr_field->type == BPF_KPTR_UNREF) verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), targ_name); else verbose(env, "\n"); return -EINVAL;}
static bool in_sleepable(struct bpf_verifier_env *env){ return env->cur_state->in_sleepable;}
/* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock() * can dereference RCU protected pointers and result is PTR_TRUSTED. */static bool in_rcu_cs(struct bpf_verifier_env *env){ return env->cur_state->active_rcu_locks || env->cur_state->active_locks || !in_sleepable(env);}
/* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */BTF_SET_START(rcu_protected_types)#ifdef CONFIG_NETBTF_ID(struct, prog_test_ref_kfunc)#endif#ifdef CONFIG_CGROUPSBTF_ID(struct, cgroup)#endif#ifdef CONFIG_BPF_JITBTF_ID(struct, bpf_cpumask)#endifBTF_ID(struct, task_struct)#ifdef CONFIG_CRYPTOBTF_ID(struct, bpf_crypto_ctx)#endifBTF_SET_END(rcu_protected_types)
static bool rcu_protected_object(const struct btf *btf, u32 btf_id){ if (!btf_is_kernel(btf)) return true; return btf_id_set_contains(&rcu_protected_types, btf_id);}
static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field){ struct btf_struct_meta *meta;
if (btf_is_kernel(kptr_field->kptr.btf)) return NULL;
meta = btf_find_struct_meta(kptr_field->kptr.btf, kptr_field->kptr.btf_id);
return meta ? meta->record : NULL;}
static bool rcu_safe_kptr(const struct btf_field *field){ const struct btf_field_kptr *kptr = &field->kptr;
return field->type == BPF_KPTR_PERCPU || (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id));}
static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field){ struct btf_record *rec; u32 ret;
ret = PTR_MAYBE_NULL; if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) { ret |= MEM_RCU; if (kptr_field->type == BPF_KPTR_PERCPU) ret |= MEM_PERCPU; else if (!btf_is_kernel(kptr_field->kptr.btf)) ret |= MEM_ALLOC;
rec = kptr_pointee_btf_record(kptr_field); if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE)) ret |= NON_OWN_REF; } else { ret |= PTR_UNTRUSTED; }
return ret;}
static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno, struct btf_field *field){ struct bpf_reg_state *reg; const struct btf_type *t;
t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id); mark_reg_known_zero(env, cur_regs(env), regno); reg = reg_state(env, regno); reg->type = PTR_TO_MEM | PTR_MAYBE_NULL; reg->mem_size = t->size; reg->id = ++env->id_gen;
return 0;}
static int check_map_kptr_access(struct bpf_verifier_env *env, int value_regno, int insn_idx, struct btf_field *kptr_field){ struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; int class = BPF_CLASS(insn->code); struct bpf_reg_state *val_reg; int ret;
/* Things we already checked for in check_map_access and caller: * - Reject cases where variable offset may touch kptr * - size of access (must be BPF_DW) * - tnum_is_const(reg->var_off) * - kptr_field->offset == off + reg->var_off.value */ /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ if (BPF_MODE(insn->code) != BPF_MEM) { verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); return -EACCES; }
/* We only allow loading referenced kptr, since it will be marked as * untrusted, similar to unreferenced kptr. */ if (class != BPF_LDX && (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) { verbose(env, "store to referenced kptr disallowed\n"); return -EACCES; } if (class != BPF_LDX && kptr_field->type == BPF_UPTR) { verbose(env, "store to uptr disallowed\n"); return -EACCES; }
if (class == BPF_LDX) { if (kptr_field->type == BPF_UPTR) return mark_uptr_ld_reg(env, value_regno, kptr_field);
/* We can simply mark the value_regno receiving the pointer * value from map as PTR_TO_BTF_ID, with the correct type. */ ret = mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field)); if (ret < 0) return ret; } else if (class == BPF_STX) { val_reg = reg_state(env, value_regno); if (!bpf_register_is_null(val_reg) && map_kptr_match_type(env, kptr_field, val_reg, value_regno)) return -EACCES; } else if (class == BPF_ST) { if (insn->imm) { verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", kptr_field->offset); return -EACCES; } } else { verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); return -EACCES; } return 0;}
/* * Return the size of the memory region accessible from a pointer to map value. * For INSN_ARRAY maps whole bpf_insn_array->ips array is accessible. */static u32 map_mem_size(const struct bpf_map *map){ if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) return map->max_entries * sizeof(long);
return map->value_size;}
/* check read/write into a map element with possible variable offset */static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, enum bpf_access_src src){ struct bpf_map *map = reg->map_ptr; u32 mem_size = map_mem_size(map); struct btf_record *rec; int err, i;
err = check_mem_region_access(env, reg, argno, off, size, mem_size, zero_size_allowed); if (err) return err;
if (IS_ERR_OR_NULL(map->record)) return 0; rec = map->record; for (i = 0; i < rec->cnt; i++) { struct btf_field *field = &rec->fields[i]; u32 p = field->offset;
/* If any part of a field can be touched by load/store, reject * this program. To check that [x1, x2) overlaps with [y1, y2), * it is sufficient to check x1 < y2 && y1 < x2. */ if (reg_smin(reg) + off < p + field->size && p < reg_umax(reg) + off + size) { switch (field->type) { case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: case BPF_UPTR: if (src != ACCESS_DIRECT) { verbose(env, "%s cannot be accessed indirectly by helper\n", btf_field_type_name(field->type)); return -EACCES; } if (!tnum_is_const(reg->var_off)) { verbose(env, "%s access cannot have variable offset\n", btf_field_type_name(field->type)); return -EACCES; } if (p != off + reg->var_off.value) { verbose(env, "%s access misaligned expected=%u off=%llu\n", btf_field_type_name(field->type), p, off + reg->var_off.value); return -EACCES; } if (size != bpf_size_to_bytes(BPF_DW)) { verbose(env, "%s access size must be BPF_DW\n", btf_field_type_name(field->type)); return -EACCES; } break; default: verbose(env, "%s cannot be accessed directly by load/store\n", btf_field_type_name(field->type)); return -EACCES; } } } return 0;}
static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta, enum bpf_access_type t){ enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
switch (prog_type) { /* Program types only with direct read access go here! */ case BPF_PROG_TYPE_LWT_IN: case BPF_PROG_TYPE_LWT_OUT: case BPF_PROG_TYPE_LWT_SEG6LOCAL: case BPF_PROG_TYPE_SK_REUSEPORT: case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_CGROUP_SKB: if (t == BPF_WRITE) return false; fallthrough;
/* Program types with direct read + write access go here! */ case BPF_PROG_TYPE_SCHED_CLS: case BPF_PROG_TYPE_SCHED_ACT: case BPF_PROG_TYPE_XDP: case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: if (meta) return meta->pkt_access;
env->seen_direct_write = true; return true;
case BPF_PROG_TYPE_CGROUP_SOCKOPT: if (t == BPF_WRITE) env->seen_direct_write = true;
return true;
default: return false; }}
static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed){ int err;
if (reg->range < 0) { verbose(env, "%s offset is outside of the packet\n", reg_arg_name(env, argno)); return -EINVAL; }
err = check_mem_region_access(env, reg, argno, off, size, reg->range, zero_size_allowed); if (err) return err;
/* __check_mem_access has made sure "off + size - 1" is within u16. * reg_umax(reg) can't be bigger than MAX_PACKET_OFF which is 0xffff, * otherwise find_good_pkt_pointers would have refused to set range info * that __check_mem_access would have rejected this pkt access. * Therefore, "off + reg_umax(reg) + size - 1" won't overflow u32. */ env->prog->aux->max_pkt_offset = max_t(u32, env->prog->aux->max_pkt_offset, off + reg_umax(reg) + size - 1);
return 0;}
static bool is_var_ctx_off_allowed(struct bpf_prog *prog){ return resolve_prog_type(prog) == BPF_PROG_TYPE_SYSCALL;}
/* check access to 'struct bpf_context' fields. Supports fixed offsets only */static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, enum bpf_access_type t, struct bpf_insn_access_aux *info){ if (env->ops->is_valid_access && env->ops->is_valid_access(off, size, t, env->prog, info)) { /* A non zero info.ctx_field_size indicates that this field is a * candidate for later verifier transformation to load the whole * field and then apply a mask when accessed with a narrower * access than actual ctx access size. A zero info.ctx_field_size * will only allow for whole field access and rejects any other * type of narrower access. */ if (base_type(info->reg_type) == PTR_TO_BTF_ID) { if (info->ref_id && !find_reference_state(env->cur_state, info->ref_id)) { verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n", off); return -EACCES; } } else { env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size; } /* remember the offset of last byte accessed in ctx */ if (env->prog->aux->max_ctx_offset < off + size) env->prog->aux->max_ctx_offset = off + size; return 0; }
verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); return -EACCES;}
static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, int off, int access_size, enum bpf_access_type t, struct bpf_insn_access_aux *info){ /* * Program types that don't rewrite ctx accesses can safely * dereference ctx pointers with fixed offsets. */ bool var_off_ok = is_var_ctx_off_allowed(env->prog); bool fixed_off_ok = !env->ops->convert_ctx_access; int err;
if (var_off_ok) err = check_mem_region_access(env, reg, argno, off, access_size, U16_MAX, false); else err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok); if (err) return err; off += reg_umax(reg);
err = __check_ctx_access(env, insn_idx, off, access_size, t, info); if (err) verbose_linfo(env, insn_idx, "; "); return err;}
static int check_flow_keys_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int size){ /* Only a constant offset is allowed here; fold it into off. */ if (!tnum_is_const(reg->var_off)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "%s invalid variable offset to flow keys: off=%d, var_off=%s\n", reg_arg_name(env, argno), off, tn_buf); return -EACCES; } off += reg->var_off.value;
if (size < 0 || off < 0 || (u64)off + size > sizeof(struct bpf_flow_keys)) { verbose(env, "invalid access to flow keys off=%d size=%d\n", off, size); return -EACCES; } return 0;}
static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, int off, int size, enum bpf_access_type t){ struct bpf_insn_access_aux info = {}; bool valid;
if (reg_smin(reg) < 0) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); return -EACCES; }
switch (reg->type) { case PTR_TO_SOCK_COMMON: valid = bpf_sock_common_is_valid_access(off, size, t, &info); break; case PTR_TO_SOCKET: valid = bpf_sock_is_valid_access(off, size, t, &info); break; case PTR_TO_TCP_SOCK: valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); break; case PTR_TO_XDP_SOCK: valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); break; default: valid = false; }
if (valid) { env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; return 0; }
verbose(env, "%s invalid %s access off=%d size=%d\n", reg_arg_name(env, argno), reg_type_str(env, reg->type), off, size);
return -EACCES;}
static bool is_pointer_value(struct bpf_verifier_env *env, int regno){ return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));}
static bool is_ctx_reg(struct bpf_verifier_env *env, int regno){ const struct bpf_reg_state *reg = reg_state(env, regno);
return reg->type == PTR_TO_CTX;}
static bool is_sk_reg(struct bpf_verifier_env *env, int regno){ const struct bpf_reg_state *reg = reg_state(env, regno);
return type_is_sk_pointer(reg->type);}
static bool is_pkt_reg(struct bpf_verifier_env *env, int regno){ const struct bpf_reg_state *reg = reg_state(env, regno);
return type_is_pkt_pointer(reg->type);}
static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno){ const struct bpf_reg_state *reg = reg_state(env, regno);
/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ return reg->type == PTR_TO_FLOW_KEYS;}
static bool is_arena_reg(struct bpf_verifier_env *env, int regno){ const struct bpf_reg_state *reg = reg_state(env, regno);
return reg->type == PTR_TO_ARENA;}
/* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn){ if (is_ctx_reg(env, regno)) return false; if (is_pkt_reg(env, regno)) return false; if (is_flow_key_reg(env, regno)) return false; if (is_sk_reg(env, regno)) return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true);
return true;}
static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {#ifdef CONFIG_NET [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],#endif [CONST_PTR_TO_MAP] = btf_bpf_map_id,};
static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg){ /* A referenced register is always trusted. */ if (reg_is_referenced(env, reg)) return true;
/* Types listed in the reg2btf_ids are always trusted */ if (reg2btf_ids[base_type(reg->type)] && !bpf_type_has_unsafe_modifiers(reg->type)) return true;
/* If a register is not referenced, it is trusted if it has the * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the * other type modifiers may be safe, but we elect to take an opt-in * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are * not. * * Eventually, we should make PTR_TRUSTED the single source of truth * for whether a register is trusted. */ return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && !bpf_type_has_unsafe_modifiers(reg->type);}
static bool is_rcu_reg(const struct bpf_reg_state *reg){ return reg->type & MEM_RCU;}
static void clear_trusted_flags(enum bpf_type_flag *flag){ *flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU);}
static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int off, int size, bool strict){ struct tnum reg_off; int ip_align;
/* Byte size accesses are always allowed. */ if (!strict || size == 1) return 0;
/* For platforms that do not have a Kconfig enabling * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of * NET_IP_ALIGN is universally set to '2'. And on platforms * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get * to this code only in strict mode where we want to emulate * the NET_IP_ALIGN==2 checking. Therefore use an * unconditional IP align value of '2'. */ ip_align = 2;
reg_off = tnum_add(reg->var_off, tnum_const(ip_align + off)); if (!tnum_is_aligned(reg_off, size)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "misaligned packet access off %d+%s+%d size %d\n", ip_align, tn_buf, off, size); return -EACCES; }
return 0;}
static int check_generic_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, const char *pointer_desc, int off, int size, bool strict){ struct tnum reg_off;
/* Byte size accesses are always allowed. */ if (!strict || size == 1) return 0;
reg_off = tnum_add(reg->var_off, tnum_const(off)); if (!tnum_is_aligned(reg_off, size)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "misaligned %saccess off %s+%d size %d\n", pointer_desc, tn_buf, off, size); return -EACCES; }
return 0;}
static int check_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int off, int size, bool strict_alignment_once){ bool strict = env->strict_alignment || strict_alignment_once; const char *pointer_desc = "";
switch (reg->type) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: /* Special case, because of NET_IP_ALIGN. Given metadata sits * right in front, treat it the very same way. */ return check_pkt_ptr_alignment(env, reg, off, size, strict); case PTR_TO_FLOW_KEYS: pointer_desc = "flow keys "; break; case PTR_TO_MAP_KEY: pointer_desc = "key "; break; case PTR_TO_MAP_VALUE: pointer_desc = "value "; if (reg->map_ptr->map_type == BPF_MAP_TYPE_INSN_ARRAY) strict = true; break; case PTR_TO_CTX: pointer_desc = "context "; break; case PTR_TO_STACK: pointer_desc = "stack "; /* The stack spill tracking logic in check_stack_write_fixed_off() * and check_stack_read_fixed_off() relies on stack accesses being * aligned. */ strict = true; break; case PTR_TO_SOCKET: pointer_desc = "sock "; break; case PTR_TO_SOCK_COMMON: pointer_desc = "sock_common "; break; case PTR_TO_TCP_SOCK: pointer_desc = "tcp_sock "; break; case PTR_TO_XDP_SOCK: pointer_desc = "xdp_sock "; break; case PTR_TO_ARENA: return 0; default: break; } return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, strict);}
static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog){ if (!bpf_jit_supports_private_stack()) return NO_PRIV_STACK;
/* bpf_prog_check_recur() checks all prog types that use bpf trampoline * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked * explicitly. */ switch (prog->type) { case BPF_PROG_TYPE_KPROBE: case BPF_PROG_TYPE_TRACEPOINT: case BPF_PROG_TYPE_PERF_EVENT: case BPF_PROG_TYPE_RAW_TRACEPOINT: return PRIV_STACK_ADAPTIVE; case BPF_PROG_TYPE_TRACING: case BPF_PROG_TYPE_LSM: case BPF_PROG_TYPE_STRUCT_OPS: if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog)) return PRIV_STACK_ADAPTIVE; fallthrough; default: break; }
return NO_PRIV_STACK;}
static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth){ if (env->prog->jit_requested) return round_up(stack_depth, 16);
/* round up to 32-bytes, since this is granularity * of interpreter stack size */ return round_up(max_t(u32, stack_depth, 1), 32);}
/* temporary state used for call frame depth calculation */struct bpf_subprog_call_depth_info { int ret_insn; /* caller instruction where we return to. */ int caller; /* caller subprogram idx */ int frame; /* # of consecutive static call stack frames on top of stack */};
/* starting from main bpf function walk all instructions of the function * and recursively walk all callees that given function can call. * Ignore jump and exit insns. */static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, struct bpf_subprog_call_depth_info *dinfo, bool priv_stack_supported){ struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_insn *insn = env->prog->insnsi; int depth = 0, frame = 0, i, subprog_end, subprog_depth; bool tail_call_reachable = false; int total; int tmp;
/* no caller idx */ dinfo[idx].caller = -1;
i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK;process_func: /* protect against potential stack overflow that might happen when * bpf2bpf calls get combined with tailcalls. Limit the caller's stack * depth for such case down to 256 so that the worst case scenario * would result in 8k stack size (32 which is tailcall limit * 256 = * 8k). * * To get the idea what might happen, see an example: * func1 -> sub rsp, 128 * subfunc1 -> sub rsp, 256 * tailcall1 -> add rsp, 256 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) * subfunc2 -> sub rsp, 64 * subfunc22 -> sub rsp, 128 * tailcall2 -> add rsp, 128 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) * * tailcall will unwind the current stack frame but it will not get rid * of caller's stack as shown on the example above. */ if (idx && subprog[idx].has_tail_call && depth >= 256) { verbose(env, "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", depth); return -EACCES; }
subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); if (IS_ENABLED(CONFIG_X86_64) && subprog[idx].stack_arg_cnt) { /* x86-64 uses R9 for both private stack frame pointer and arg6. */ subprog[idx].priv_stack_mode = NO_PRIV_STACK; } else if (priv_stack_supported) { /* Request private stack support only if the subprog stack * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to * avoid jit penalty if the stack usage is small. */ if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN && subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE; }
if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { if (subprog_depth > env->max_stack_depth) env->max_stack_depth = subprog_depth; if (subprog_depth > MAX_BPF_STACK) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); return -EACCES; } } else { depth += subprog_depth; if (depth > env->max_stack_depth) env->max_stack_depth = depth; if (depth > MAX_BPF_STACK) { total = 0; for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) total++;
verbose(env, "combined stack size of %d calls is %d. Too large\n", total, depth); return -EACCES; } }continue_func: subprog_end = subprog[idx + 1].start; for (; i < subprog_end; i++) { int next_insn, sidx;
if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { bool err = false;
if (!bpf_is_throw_kfunc(insn + i)) continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { err = true; break; } } if (!err) continue; verbose(env, "bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n", i, idx); return -EINVAL; }
if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) continue; /* remember insn and function to return to */
/* find the callee */ next_insn = i + insn[i].imm + 1; sidx = bpf_find_subprog(env, next_insn); if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) return -EFAULT; if (subprog[sidx].is_async_cb) { if (subprog[sidx].has_tail_call) { verifier_bug(env, "subprog has tail_call and async cb"); return -EFAULT; } /* async callbacks don't increase bpf prog stack size unless called directly */ if (!bpf_pseudo_call(insn + i)) continue; if (subprog[sidx].is_exception_cb) { verbose(env, "insn %d cannot call exception cb directly", i); return -EINVAL; } }
/* store caller info for after we return from callee */ dinfo[idx].frame = frame; dinfo[idx].ret_insn = i + 1;
/* push caller idx into callee's dinfo */ dinfo[sidx].caller = idx;
i = next_insn;
idx = sidx; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK;
if (subprog[idx].has_tail_call) tail_call_reachable = true;
frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { verbose(env, "the call stack of %d frames is too deep !\n", frame); return -E2BIG; } goto process_func; } /* if tail call got detected across bpf2bpf calls then mark each of the * currently present subprog frames as tail call reachable subprogs; * this info will be utilized by JIT so that we will be preserving the * tail call counter throughout bpf2bpf calls combined with tailcalls */ if (tail_call_reachable) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_exception_cb) { verbose(env, "cannot tail call within exception cb\n"); return -EINVAL; } if (subprog[tmp].stack_arg_cnt) { verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; } subprog[tmp].tail_call_reachable = true; } } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; }
if (subprog[0].tail_call_reachable) env->prog->aux->tail_call_reachable = true;
/* end of for() loop means the last insn of the 'subprog' * was reached. Doesn't matter whether it was JA or EXIT */ if (frame == 0 && dinfo[idx].caller < 0) return 0; if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE) depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
/* pop caller idx from callee */ idx = dinfo[idx].caller;
/* retrieve caller state from its frame */ frame = dinfo[idx].frame; i = dinfo[idx].ret_insn;
/* reset tail_call_reachable to the parent's actual state */ tail_call_reachable = subprog[idx].tail_call_reachable;
goto continue_func;}
static int check_max_stack_depth(struct bpf_verifier_env *env){ enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN; struct bpf_subprog_call_depth_info *dinfo; struct bpf_subprog_info *si = env->subprog_info; bool priv_stack_supported; int ret;
dinfo = kvcalloc(env->subprog_cnt, sizeof(*dinfo), GFP_KERNEL_ACCOUNT); if (!dinfo) return -ENOMEM;
for (int i = 0; i < env->subprog_cnt; i++) { if (si[i].has_tail_call) { priv_stack_mode = NO_PRIV_STACK; break; } }
if (priv_stack_mode == PRIV_STACK_UNKNOWN) priv_stack_mode = bpf_enable_priv_stack(env->prog);
/* All async_cb subprogs use normal kernel stack. If a particular * subprog appears in both main prog and async_cb subtree, that * subprog will use normal kernel stack to avoid potential nesting. * The reverse subprog traversal ensures when main prog subtree is * checked, the subprogs appearing in async_cb subtrees are already * marked as using normal kernel stack, so stack size checking can * be done properly. */ for (int i = env->subprog_cnt - 1; i >= 0; i--) { if (!i || si[i].is_async_cb) { priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE; ret = check_max_stack_depth_subprog(env, i, dinfo, priv_stack_supported); if (ret < 0) { kvfree(dinfo); return ret; } } }
for (int i = 0; i < env->subprog_cnt; i++) { if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) { env->prog->aux->jits_use_priv_stack = true; break; } }
kvfree(dinfo);
return 0;}
static int __check_buffer_access(struct bpf_verifier_env *env, const char *buf_info, const struct bpf_reg_state *reg, argno_t argno, int off, int size, u32 *access_end){ s64 start;
if (!tnum_is_const(reg->var_off)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "%s invalid variable buffer offset: off=%d, var_off=%s\n", reg_arg_name(env, argno), off, tn_buf); return -EACCES; }
start = (s64)reg->var_off.value + off; if (start < 0) { verbose(env, "%s invalid negative %s buffer offset: off=%d, var_off=%lld\n", reg_arg_name(env, argno), buf_info, off, (s64)reg->var_off.value); return -EACCES; }
*access_end = start + size; return 0;}
static int check_tp_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, int off, int size){ u32 access_end; int err;
err = __check_buffer_access(env, "tracepoint", reg, argno, off, size, &access_end); if (err) return err;
env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access);
return 0;}
static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, u32 *max_access){ const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; u32 access_end; int err;
err = __check_buffer_access(env, buf_info, reg, argno, off, size, &access_end); if (err) return err;
*max_access = max(access_end, *max_access);
return 0;}
/* BPF architecture zero extends alu32 ops into 64-bit registesr */static void zext_32_to_64(struct bpf_reg_state *reg){ reg->var_off = tnum_subreg(reg->var_off); reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg));}
/* truncate register to smaller size (in bytes) * must be called with size < BPF_REG_SIZE */static void coerce_reg_to_size(struct bpf_reg_state *reg, int size){ u64 mask;
/* clear high bits in bit representation */ reg->var_off = tnum_cast(reg->var_off, size);
/* fix arithmetic bounds */ mask = ((u64)1 << (size * 8)) - 1; if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); else reg_set_urange64(reg, 0, mask);
/* If size is smaller than 32bit register the 32bit register * values are also truncated so we push 64-bit bounds into * 32-bit bounds. Above were truncated < 32-bits already. */ if (size < 4) __mark_reg32_unbounded(reg);
reg_bounds_sync(reg);}
static void set_sext64_default_val(struct bpf_reg_state *reg, int size){ if (size == 1) { reg_set_srange64(reg, S8_MIN, S8_MAX); reg_set_srange32(reg, S8_MIN, S8_MAX); } else if (size == 2) { reg_set_srange64(reg, S16_MIN, S16_MAX); reg_set_srange32(reg, S16_MIN, S16_MAX); } else { /* size == 4 */ reg_set_srange64(reg, S32_MIN, S32_MAX); reg_set_srange32(reg, S32_MIN, S32_MAX); } reg->var_off = tnum_unknown;}
static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size){ s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; u64 top_smax_value, top_smin_value; u64 num_bits = size * 8;
if (tnum_is_const(reg->var_off)) { u64_cval = reg->var_off.value; if (size == 1) reg->var_off = tnum_const((s8)u64_cval); else if (size == 2) reg->var_off = tnum_const((s16)u64_cval); else /* size == 4 */ reg->var_off = tnum_const((s32)u64_cval);
u64_cval = reg->var_off.value; reg->r64 = cnum64_from_urange(u64_cval, u64_cval); reg->r32 = cnum32_from_urange((u32)u64_cval, (u32)u64_cval); return; }
top_smax_value = ((u64)reg_smax(reg) >> num_bits) << num_bits; top_smin_value = ((u64)reg_smin(reg) >> num_bits) << num_bits;
if (top_smax_value != top_smin_value) goto out;
/* find the s64_min and s64_min after sign extension */ if (size == 1) { init_s64_max = (s8)reg_smax(reg); init_s64_min = (s8)reg_smin(reg); } else if (size == 2) { init_s64_max = (s16)reg_smax(reg); init_s64_min = (s16)reg_smin(reg); } else { init_s64_max = (s32)reg_smax(reg); init_s64_min = (s32)reg_smin(reg); }
s64_max = max(init_s64_max, init_s64_min); s64_min = min(init_s64_max, init_s64_min);
/* both of s64_max/s64_min positive or negative */ if ((s64_max >= 0) == (s64_min >= 0)) { reg_set_srange64(reg, s64_min, s64_max); reg_set_srange32(reg, s64_min, s64_max); reg->var_off = tnum_range(s64_min, s64_max); return; }
out: set_sext64_default_val(reg, size);}
static void set_sext32_default_val(struct bpf_reg_state *reg, int size){ if (size == 1) reg_set_srange32(reg, S8_MIN, S8_MAX); else /* size == 2 */ reg_set_srange32(reg, S16_MIN, S16_MAX); reg->var_off = tnum_subreg(tnum_unknown);}
static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size){ s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val; u32 top_smax_value, top_smin_value; u32 num_bits = size * 8;
if (tnum_is_const(reg->var_off)) { u32_val = reg->var_off.value; if (size == 1) reg->var_off = tnum_const((s8)u32_val); else reg->var_off = tnum_const((s16)u32_val);
u32_val = reg->var_off.value; reg_set_srange32(reg, u32_val, u32_val); return; }
top_smax_value = ((u32)reg_s32_max(reg) >> num_bits) << num_bits; top_smin_value = ((u32)reg_s32_min(reg) >> num_bits) << num_bits;
if (top_smax_value != top_smin_value) goto out;
/* find the s32_min and s32_min after sign extension */ if (size == 1) { init_s32_max = (s8)reg_s32_max(reg); init_s32_min = (s8)reg_s32_min(reg); } else { /* size == 2 */ init_s32_max = (s16)reg_s32_max(reg); init_s32_min = (s16)reg_s32_min(reg); } s32_max = max(init_s32_max, init_s32_min); s32_min = min(init_s32_max, init_s32_min);
if ((s32_min >= 0) == (s32_max >= 0)) { reg_set_srange32(reg, s32_min, s32_max); reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max)); return; }
out: set_sext32_default_val(reg, size);}
bool bpf_map_is_rdonly(const struct bpf_map *map){ /* A map is considered read-only if the following condition are true: * * 1) BPF program side cannot change any of the map content. The * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map * and was set at map creation time. * 2) The map value(s) have been initialized from user space by a * loader and then "frozen", such that no new map update/delete * operations from syscall side are possible for the rest of * the map's lifetime from that point onwards. * 3) Any parallel/pending map update/delete operations from syscall * side have been completed. Only after that point, it's safe to * assume that map value(s) are immutable. */ return (map->map_flags & BPF_F_RDONLY_PROG) && READ_ONCE(map->frozen) && !bpf_map_write_active(map);}
int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx){ void *ptr; u64 addr; int err;
err = map->ops->map_direct_value_addr(map, &addr, off); if (err) return err; ptr = (void *)(long)addr + off;
switch (size) { case sizeof(u8): *val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr; break; case sizeof(u16): *val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr; break; case sizeof(u32): *val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr; break; case sizeof(u64): *val = *(u64 *)ptr; break; default: return -EINVAL; } return 0;}
#define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu)#define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null)#define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted)#define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type) __PASTE(__type, __safe_trusted_or_null)
/* * Allow list few fields as RCU trusted or full trusted. * This logic doesn't allow mix tagging and will be removed once GCC supports * btf_type_tag. */
/* RCU trusted: these fields are trusted in RCU CS and never NULL */BTF_TYPE_SAFE_RCU(struct task_struct) { const cpumask_t *cpus_ptr; struct css_set __rcu *cgroups; struct task_struct __rcu *real_parent; struct task_struct *group_leader;};
BTF_TYPE_SAFE_RCU(struct cgroup) { /* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */ struct kernfs_node *kn;};
BTF_TYPE_SAFE_RCU(struct css_set) { struct cgroup *dfl_cgrp;};
BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state) { struct cgroup *cgroup;};
/* RCU trusted: these fields are trusted in RCU CS and can be NULL */BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) { struct file __rcu *exe_file;#ifdef CONFIG_MEMCG struct task_struct __rcu *owner;#endif};
/* skb->sk, req->sk are not RCU protected, but we mark them as such * because bpf prog accessible sockets are SOCK_RCU_FREE. */BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) { struct sock *sk;};
BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) { struct sock *sk;};
/* full trusted: these fields are trusted even outside of RCU CS and never NULL */BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) { struct seq_file *seq;};
BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) { struct bpf_iter_meta *meta; struct task_struct *task;};
BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) { struct file *file;};
BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode;};
BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode;};
BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { struct sock *sk;};
BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct) { struct mm_struct *vm_mm; struct file *vm_file;};
static bool type_is_rcu(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *field_name, u32 btf_id){ BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state));
return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");}
static bool type_is_rcu_or_null(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *field_name, u32 btf_id){ BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock));
return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");}
static bool type_is_trusted(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *field_name, u32 btf_id){ BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");}
static bool type_is_trusted_or_null(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const char *field_name, u32 btf_id){ BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct));
return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted_or_null");}
static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, enum bpf_access_type atype, int value_regno){ const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); const char *tname = btf_name_by_offset(reg->btf, t->name_off); const char *field_name = NULL; enum bpf_type_flag flag = 0; u32 btf_id = 0; int ret;
if (!env->allow_ptr_leaks) { verbose(env, "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", tname); return -EPERM; } if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { verbose(env, "Cannot access kernel 'struct %s' from non-GPL compatible program\n", tname); return -EINVAL; }
if (!tnum_is_const(reg->var_off)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", reg_arg_name(env, argno), tname, off, tn_buf); return -EACCES; }
off += reg->var_off.value;
if (off < 0) { verbose(env, "%s is ptr_%s invalid negative access: off=%d\n", reg_arg_name(env, argno), tname, off); return -EACCES; }
if (reg->type & MEM_USER) { verbose(env, "%s is ptr_%s access user memory: off=%d\n", reg_arg_name(env, argno), tname, off); return -EACCES; }
if (reg->type & MEM_PERCPU) { verbose(env, "%s is ptr_%s access percpu memory: off=%d\n", reg_arg_name(env, argno), tname, off); return -EACCES; }
if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); return -EFAULT; } ret = env->ops->btf_struct_access(&env->log, reg, off, size); if (ret < 0) verbose(env, "%s cannot write into ptr_%s at off=%d size=%d\n", reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for * program allocated objects (which always have id > 0), * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { verbose(env, "only read is supported\n"); return -EACCES; }
if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) { verifier_bug(env, "allocated object must have a referenced id"); return -EFAULT; }
ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); }
if (ret < 0) return ret;
if (ret != PTR_TO_BTF_ID) { /* just mark; */
} else if (type_flag(reg->type) & PTR_UNTRUSTED) { /* If this is an untrusted pointer, all pointers formed by walking it * also inherit the untrusted flag. */ flag = PTR_UNTRUSTED;
} else if (is_trusted_reg(env, reg) || is_rcu_reg(reg)) { /* By default any pointer obtained from walking a trusted pointer is no * longer trusted, unless the field being accessed has explicitly been * marked as inheriting its parent's state of trust (either full or RCU). * For example: * 'cgroups' pointer is untrusted if task->cgroups dereference * happened in a sleepable program outside of bpf_rcu_read_lock() * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU). * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED. * * A regular RCU-protected pointer with __rcu tag can also be deemed * trusted if we are in an RCU CS. Such pointer can be NULL. */ if (type_is_trusted(env, reg, field_name, btf_id)) { flag |= PTR_TRUSTED; } else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) { flag |= PTR_TRUSTED | PTR_MAYBE_NULL; } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { if (type_is_rcu(env, reg, field_name, btf_id)) { /* ignore __rcu tag and mark it MEM_RCU */ flag |= MEM_RCU; } else if (flag & MEM_RCU || type_is_rcu_or_null(env, reg, field_name, btf_id)) { /* __rcu tagged pointers can be NULL */ flag |= MEM_RCU | PTR_MAYBE_NULL;
/* We always trust them */ if (type_is_rcu_or_null(env, reg, field_name, btf_id) && flag & PTR_UNTRUSTED) flag &= ~PTR_UNTRUSTED; } else if (flag & (MEM_PERCPU | MEM_USER)) { /* keep as-is */ } else { /* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */ clear_trusted_flags(&flag); } } else { /* * If not in RCU CS or MEM_RCU pointer can be NULL then * aggressively mark as untrusted otherwise such * pointers will be plain PTR_TO_BTF_ID without flags * and will be allowed to be passed into helpers for * compat reasons. */ flag = PTR_UNTRUSTED; } } else { /* Old compat. Deprecated */ clear_trusted_flags(&flag); }
if (atype == BPF_READ && value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) return ret; }
return 0;}
static int check_ptr_to_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, enum bpf_access_type atype, int value_regno){ struct bpf_map *map = reg->map_ptr; struct bpf_reg_state map_reg; enum bpf_type_flag flag = 0; const struct btf_type *t; const char *tname; u32 btf_id; int ret;
if (!btf_vmlinux) { verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); return -ENOTSUPP; }
if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { verbose(env, "map_ptr access not supported for map type %d\n", map->map_type); return -ENOTSUPP; }
t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); tname = btf_name_by_offset(btf_vmlinux, t->name_off);
if (!env->allow_ptr_leaks) { verbose(env, "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", tname); return -EPERM; }
if (off < 0) { verbose(env, "%s is %s invalid negative access: off=%d\n", reg_arg_name(env, argno), tname, off); return -EACCES; }
if (atype != BPF_READ) { verbose(env, "only read from %s is supported\n", tname); return -EACCES; }
/* Simulate access to a PTR_TO_BTF_ID */ memset(&map_reg, 0, sizeof(map_reg)); ret = mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); if (ret < 0) return ret; ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); if (ret < 0) return ret;
if (value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); if (ret < 0) return ret; }
return 0;}
/* Check that the stack access at the given offset is within bounds. The * maximum valid offset is -1. * * The minimum valid offset is -MAX_BPF_STACK for writes, and * -state->allocated_stack for reads. */static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, s64 off, struct bpf_func_state *state, enum bpf_access_type t){ int min_valid_off;
if (t == BPF_WRITE || env->allow_uninit_stack) min_valid_off = -MAX_BPF_STACK; else min_valid_off = -state->allocated_stack;
if (off < min_valid_off || off > -1) return -EACCES; return 0;}
/* Check that the stack access at 'regno + off' falls within the maximum stack * bounds. * * 'off' includes `regno->offset`, but not its dynamic part (if any). */static int check_stack_access_within_bounds( struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int access_size, enum bpf_access_type type){ struct bpf_func_state *state = bpf_func(env, reg); s64 min_off, max_off; int err; char *err_extra;
if (type == BPF_READ) err_extra = " read from"; else err_extra = " write to";
if (tnum_is_const(reg->var_off)) { min_off = (s64)reg->var_off.value + off; max_off = min_off + access_size; } else { if (reg_smax(reg) >= BPF_MAX_VAR_OFF || reg_smin(reg) <= -BPF_MAX_VAR_OFF) { verbose(env, "invalid unbounded variable-offset%s stack %s\n", err_extra, reg_arg_name(env, argno)); return -EACCES; } min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off + access_size; }
err = check_stack_slot_within_bounds(env, min_off, state, type); if (!err && max_off > 0) err = -EINVAL; /* out of stack access into non-negative offsets */ if (!err && access_size < 0) /* access_size should not be negative (or overflow an int); others checks * along the way should have prevented such an access. */ err = -EFAULT; /* invalid negative access size; integer overflow? */
if (err) { if (tnum_is_const(reg->var_off)) { verbose(env, "invalid%s stack %s off=%lld size=%d\n", err_extra, reg_arg_name(env, argno), min_off, access_size); } else { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "invalid variable-offset%s stack %s var_off=%s off=%d size=%d\n", err_extra, reg_arg_name(env, argno), tn_buf, off, access_size); } return err; }
/* Note that there is no stack access with offset zero, so the needed stack * size is -min_off, not -min_off+1. */ return grow_stack_state(env, state, -min_off /* size */);}
static bool get_func_retval_range(struct bpf_prog *prog, struct bpf_retval_range *range){ if (prog->type == BPF_PROG_TYPE_LSM && prog->expected_attach_type == BPF_LSM_MAC && !bpf_lsm_get_retval_range(prog, range)) { return true; } return false;}
static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val){ struct bpf_reg_state fake_reg;
if (!val) return;
fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, val);
scalar32_min_max_add(dst_reg, &fake_reg); scalar_min_max_add(dst_reg, &fake_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off);
reg_bounds_sync(dst_reg);}
/* check whether memory at (regno + off) is accessible for t = (read | write) * if t==write, value_regno is a register which value is stored into memory * if t==read, value_regno is a register which will receive the value from memory * if t==write && value_regno==-1, some unknown value is stored into memory * if t==read && value_regno==-1, don't care what we read from memory */static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, int off, int bpf_size, enum bpf_access_type t, int value_regno, bool strict_alignment_once, bool is_ldsx){ struct bpf_reg_state *regs = cur_regs(env); int size, err = 0;
size = bpf_size_to_bytes(bpf_size); if (size < 0) return size;
err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); if (err) return err;
if (reg->type == PTR_TO_MAP_KEY) { if (t == BPF_WRITE) { verbose(env, "write to change key %s not allowed\n", reg_arg_name(env, argno)); return -EACCES; }
err = check_mem_region_access(env, reg, argno, off, size, reg->map_ptr->key_size, false); if (err) return err; if (value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_MAP_VALUE) { struct btf_field *kptr_field = NULL;
if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { verbose(env, "R%d leaks addr into map\n", value_regno); return -EACCES; } err = check_map_access_type(env, reg, off, size, t); if (err) return err; err = check_map_access(env, reg, argno, off, size, false, ACCESS_DIRECT); if (err) return err; if (tnum_is_const(reg->var_off)) kptr_field = btf_record_find(reg->map_ptr->record, off + reg->var_off.value, BPF_KPTR | BPF_UPTR); if (kptr_field) { err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field); } else if (t == BPF_READ && value_regno >= 0) { struct bpf_map *map = reg->map_ptr;
/* * If map is read-only, track its contents as scalars, * unless it is an insn array (see the special case below) */ if (tnum_is_const(reg->var_off) && bpf_map_is_rdonly(map) && map->ops->map_direct_value_addr && map->map_type != BPF_MAP_TYPE_INSN_ARRAY) { int map_off = off + reg->var_off.value; u64 val = 0;
err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); if (err) return err;
regs[value_regno].type = SCALAR_VALUE; __mark_reg_known(®s[value_regno], val); } else if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { if (bpf_size != BPF_DW) { verbose(env, "Invalid read of %d bytes from insn_array\n", size); return -EACCES; } regs[value_regno] = *reg; add_scalar_to_reg(®s[value_regno], off); regs[value_regno].type = PTR_TO_INSN; } else { mark_reg_unknown(env, regs, value_regno); } } } else if (base_type(reg->type) == PTR_TO_MEM) { bool rdonly_mem = type_is_rdonly_mem(reg->type); bool rdonly_untrusted = rdonly_mem && (reg->type & PTR_UNTRUSTED);
if (type_may_be_null(reg->type)) { verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; }
if (t == BPF_WRITE && rdonly_mem) { verbose(env, "%s cannot write into %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; }
if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { verbose(env, "R%d leaks addr into mem\n", value_regno); return -EACCES; }
/* * Accesses to untrusted PTR_TO_MEM are done through probe * instructions, hence no need to check bounds in that case. */ if (!rdonly_untrusted) err = check_mem_region_access(env, reg, argno, off, size, reg->mem_size, false); if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_CTX) { struct bpf_insn_access_aux info = { .reg_type = SCALAR_VALUE, .is_ldsx = is_ldsx, .log = &env->log, }; struct bpf_retval_range range;
if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { verbose(env, "R%d leaks addr into ctx\n", value_regno); return -EACCES; }
err = check_ctx_access(env, insn_idx, reg, argno, off, size, t, &info); if (!err && t == BPF_READ && value_regno >= 0) { /* ctx access returns either a scalar, or a * PTR_TO_PACKET[_META,_END]. In the latter * case, we know the offset is zero. */ if (info.reg_type == SCALAR_VALUE) { if (info.is_retval && get_func_retval_range(env->prog, &range)) { mark_reg_unknown(env, regs, value_regno); err = __mark_reg_s32_range(env, regs, value_regno, range.minval, range.maxval); if (err) return err; } else { mark_reg_unknown(env, regs, value_regno); } } else { mark_reg_known_zero(env, regs, value_regno); /* A load of ctx field could have different * actual load size with the one encoded in the * insn. When the dst is PTR, it is for sure not * a sub-register. */ regs[value_regno].subreg_def = DEF_NOT_SUBREG; if (base_type(info.reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; regs[value_regno].id = info.ref_id; } if (type_may_be_null(info.reg_type) && !regs[value_regno].id) regs[value_regno].id = ++env->id_gen; } regs[value_regno].type = info.reg_type; }
} else if (reg->type == PTR_TO_STACK) { /* Basic bounds checks. */ err = check_stack_access_within_bounds(env, reg, argno, off, size, t); if (err) return err;
if (t == BPF_READ) err = check_stack_read(env, reg, argno, off, size, value_regno); else err = check_stack_write(env, reg, off, size, value_regno, insn_idx); } else if (reg_is_pkt_pointer(reg)) { if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { verbose(env, "cannot write into packet\n"); return -EACCES; } if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { verbose(env, "R%d leaks addr into packet\n", value_regno); return -EACCES; } err = check_packet_access(env, reg, argno, off, size, false); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_FLOW_KEYS) { if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) { verbose(env, "R%d leaks addr into flow keys\n", value_regno); return -EACCES; }
err = check_flow_keys_access(env, reg, argno, off, size); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (type_is_sk_pointer(reg->type)) { if (t == BPF_WRITE) { verbose(env, "%s cannot write into %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } err = check_sock_access(env, insn_idx, reg, argno, off, size, t); if (!err && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_TP_BUFFER) { err = check_tp_buffer_access(env, reg, argno, off, size); if (!err && t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else if (base_type(reg->type) == PTR_TO_BTF_ID && !type_may_be_null(reg->type)) { err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t, value_regno); } else if (reg->type == CONST_PTR_TO_MAP) { err = check_ptr_to_map_access(env, regs, reg, argno, off, size, t, value_regno); } else if (base_type(reg->type) == PTR_TO_BUF && !type_may_be_null(reg->type)) { bool rdonly_mem = type_is_rdonly_mem(reg->type); u32 *max_access;
if (rdonly_mem) { if (t == BPF_WRITE) { verbose(env, "%s cannot write into %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } max_access = &env->prog->aux->max_rdonly_access; } else { max_access = &env->prog->aux->max_rdwr_access; }
err = check_buffer_access(env, reg, argno, off, size, false, max_access);
if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); } else if (reg->type == PTR_TO_ARENA) { if (t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else { verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; }
if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && regs[value_regno].type == SCALAR_VALUE) { if (!is_ldsx) /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); else coerce_reg_to_size_sx(®s[value_regno], size); } return err;}
static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch);
static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); enum bpf_reg_type src_reg_type; int err;
/* Handle stack arg read */ if (is_stack_arg_ldx(insn)) { err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); if (err) return err; return check_stack_arg_read(env, state, insn->off, insn->dst_reg); }
/* check src operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err;
/* check dst operand */ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); if (err) return err;
src_reg_type = regs[insn->src_reg].type;
/* Check if (src_reg + off) is readable. The state of dst_reg will be * updated by this call. */ err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx);
return err;}
static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); enum bpf_reg_type dst_reg_type; int err;
/* Handle stack arg write */ if (is_stack_arg_stx(insn)) { err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; return check_stack_arg_write(env, state, insn->off, regs + insn->src_reg); }
/* check src1 operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err;
/* check src2 operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err;
dst_reg_type = regs[insn->dst_reg].type;
/* Check if (dst_reg + off) is writeable. */ err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false);
return err;}
static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_reg_state *dst_reg; int load_reg; int err;
if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { verbose(env, "invalid atomic operand size\n"); return -EINVAL; }
/* check src1 operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err;
/* check src2 operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err;
if (insn->imm == BPF_CMPXCHG) { /* Check comparison of R0 with memory location */ const u32 aux_reg = BPF_REG_0;
err = check_reg_arg(env, aux_reg, SRC_OP); if (err) return err;
if (is_pointer_value(env, aux_reg)) { verbose(env, "R%d leaks addr into mem\n", aux_reg); return -EACCES; } }
if (is_pointer_value(env, insn->src_reg)) { verbose(env, "R%d leaks addr into mem\n", insn->src_reg); return -EACCES; }
if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", insn->dst_reg, reg_type_str(env, reg_state(env, insn->dst_reg)->type)); return -EACCES; }
if (insn->imm & BPF_FETCH) { if (insn->imm == BPF_CMPXCHG) load_reg = BPF_REG_0; else load_reg = insn->src_reg;
/* check and record load of old value */ err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; } else { /* This instruction accesses a memory location but doesn't * actually load it into a register. */ load_reg = -1; }
dst_reg = cur_regs(env) + insn->dst_reg;
/* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); if (!err && load_reg >= 0) err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); if (err) return err;
if (is_arena_reg(env, insn->dst_reg)) { err = save_aux_ptr_type(env, PTR_TO_ARENA, false); if (err) return err; } /* Check whether we can write into the same memory. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); if (err) return err; return 0;}
static int check_atomic_load(struct bpf_verifier_env *env, struct bpf_insn *insn){ int err;
err = check_load_mem(env, insn, true, false, false, "atomic_load"); if (err) return err;
if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) { verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n", insn->src_reg, reg_type_str(env, reg_state(env, insn->src_reg)->type)); return -EACCES; }
return 0;}
static int check_atomic_store(struct bpf_verifier_env *env, struct bpf_insn *insn){ int err;
err = check_store_reg(env, insn, true); if (err) return err;
if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", insn->dst_reg, reg_type_str(env, reg_state(env, insn->dst_reg)->type)); return -EACCES; }
return 0;}
static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn){ switch (insn->imm) { case BPF_ADD: case BPF_ADD | BPF_FETCH: case BPF_AND: case BPF_AND | BPF_FETCH: case BPF_OR: case BPF_OR | BPF_FETCH: case BPF_XOR: case BPF_XOR | BPF_FETCH: case BPF_XCHG: case BPF_CMPXCHG: return check_atomic_rmw(env, insn); case BPF_LOAD_ACQ: if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) { verbose(env, "64-bit load-acquires are only supported on 64-bit arches\n"); return -EOPNOTSUPP; } return check_atomic_load(env, insn); case BPF_STORE_REL: if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) { verbose(env, "64-bit store-releases are only supported on 64-bit arches\n"); return -EOPNOTSUPP; } return check_atomic_store(env, insn); default: verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); return -EINVAL; }}
/* When register 'regno' is used to read the stack (either directly or through * a helper function) make sure that it's within stack boundary and, depending * on the access type and privileges, that all elements of the stack are * initialized. * * All registers that have been spilled on the stack in the slots within the * read offsets are marked as read. */static int check_stack_range_initialized( struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, int access_size, bool zero_size_allowed, enum bpf_access_type type, struct bpf_call_arg_meta *meta){ struct bpf_func_state *state = bpf_func(env, reg); int err, min_off, max_off, i, j, slot, spi; /* Some accesses can write anything into the stack, others are * read-only. */ bool clobber = type == BPF_WRITE; /* * Negative access_size signals global subprog/kfunc arg check where * STACK_POISON slots are acceptable. static stack liveness * might have determined that subprog doesn't read them, * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber;
access_size = abs(access_size);
if (access_size == 0 && !zero_size_allowed) { verbose(env, "invalid zero-sized read\n"); return -EACCES; }
err = check_stack_access_within_bounds(env, reg, argno, off, access_size, type); if (err) return err;
if (tnum_is_const(reg->var_off)) { min_off = max_off = reg->var_off.value + off; } else { /* Variable offset is prohibited for unprivileged mode for * simplicity since it requires corresponding support in * Spectre masking for stack ALU. * See also retrieve_ptr_limit(). */ if (!env->bypass_spec_v1) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "%s variable offset stack access prohibited for !root, var_off=%s\n", reg_arg_name(env, argno), tn_buf); return -EACCES; } /* Only initialized buffer on stack is allowed to be accessed * with variable offset. With uninitialized buffer it's hard to * guarantee that whole memory is marked as initialized on * helper return since specific bounds are unknown what may * cause uninitialized stack leaking. */ if (meta && meta->raw_mode) meta = NULL;
min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; }
if (meta && meta->raw_mode) { /* Ensure we won't be overwriting dynptrs when simulating byte * by byte access in check_helper_call using meta.access_size. * This would be a problem if we have a helper in the future * which takes: * * helper(uninit_mem, len, dynptr) * * Now, uninint_mem may overlap with dynptr pointer. Hence, it * may end up writing to dynptr itself when touching memory from * arg 1. This can be relaxed on a case by case basis for known * safe cases, but reject due to the possibilitiy of aliasing by * default. */ for (i = min_off; i < max_off + access_size; i++) { int stack_off = -i - 1;
spi = bpf_get_spi(i); /* raw_mode may write past allocated_stack */ if (state->allocated_stack <= stack_off) continue; if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { verbose(env, "potential write to dynptr at off=%d disallowed\n", i); return -EACCES; } } meta->access_size = access_size; meta->regno = reg_from_argno(argno); return 0; }
for (i = min_off; i < max_off + access_size; i++) { u8 *stype;
slot = -i - 1; spi = slot / BPF_REG_SIZE; if (state->allocated_stack <= slot) { verbose(env, "allocated_stack too small\n"); return -EFAULT; }
stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || (*stype == STACK_INVALID && env->allow_uninit_stack)) { if (clobber) { /* helper can write anything into the stack */ *stype = STACK_MISC; } goto mark; }
if (bpf_is_spilled_reg(&state->stack[spi]) && (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { if (clobber) { __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); for (j = 0; j < BPF_REG_SIZE; j++) scrub_spilled_slot(&state->stack[spi].slot_type[j]); } goto mark; }
if (*stype == STACK_POISON) { if (allow_poison) goto mark; verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { verbose(env, "invalid read from stack %s off %d+%d size %d\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "invalid read from stack %s var_off %s+%d size %d\n", reg_arg_name(env, argno), tn_buf, i - min_off, access_size); } return -EACCES;mark: ; } return 0;}
static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int access_size, enum bpf_access_type access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta){ struct bpf_reg_state *regs = cur_regs(env); u32 *max_access;
switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: return check_packet_access(env, reg, argno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: if (access_type == BPF_WRITE) { verbose(env, "%s cannot write into %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } return check_mem_region_access(env, reg, argno, 0, access_size, reg->map_ptr->key_size, false); case PTR_TO_MAP_VALUE: if (check_map_access_type(env, reg, 0, access_size, access_type)) return -EACCES; return check_map_access(env, reg, argno, 0, access_size, zero_size_allowed, ACCESS_HELPER); case PTR_TO_MEM: if (type_is_rdonly_mem(reg->type)) { if (access_type == BPF_WRITE) { verbose(env, "%s cannot write into %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } } return check_mem_region_access(env, reg, argno, 0, access_size, reg->mem_size, zero_size_allowed); case PTR_TO_BUF: if (type_is_rdonly_mem(reg->type)) { if (access_type == BPF_WRITE) { verbose(env, "%s cannot write into %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; }
max_access = &env->prog->aux->max_rdonly_access; } else { max_access = &env->prog->aux->max_rdwr_access; } return check_buffer_access(env, reg, argno, 0, access_size, zero_size_allowed, max_access); case PTR_TO_STACK: return check_stack_range_initialized( env, reg, argno, 0, access_size, zero_size_allowed, access_type, meta); case PTR_TO_BTF_ID: return check_ptr_to_btf_access(env, regs, reg, argno, 0, access_size, access_type, -1); case PTR_TO_CTX: /* Only permit reading or writing syscall context using helper calls. */ if (is_var_ctx_off_allowed(env->prog)) { int err = check_mem_region_access(env, reg, argno, 0, access_size, U16_MAX, zero_size_allowed); if (err) return err; if (env->prog->aux->max_ctx_offset < reg_umax(reg) + access_size) env->prog->aux->max_ctx_offset = reg_umax(reg) + access_size; return 0; } fallthrough; default: /* scalar_value or invalid ptr */ /* Allow zero-byte read from NULL, regardless of pointer type */ if (zero_size_allowed && access_size == 0 && bpf_register_is_null(reg)) return 0;
verbose(env, "%s type=%s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); return -EACCES; }}
/* verify arguments to helpers or kfuncs consisting of a pointer and an access * size. * * @mem_reg contains the pointer, @size_reg contains the access size. */static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno, enum bpf_access_type access_type, bool zero_size_allowed, struct bpf_call_arg_meta *meta){ int err;
/* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. * See do_refine_retval_range() for helpers that can refine * the return value. C type of helper is u32 so we pull register * bound from umax_value however, if negative verifier errors * out. Only upper bounds can be learned because retval is an * int type and negative retvals are allowed. */ meta->msize_max_value = reg_umax(size_reg);
/* The register is SCALAR_VALUE; the access check happens using * its boundaries. For unprivileged variable accesses, disable * raw mode so that the program is required to initialize all * the memory that the helper could just partially fill up. */ if (!tnum_is_const(size_reg->var_off)) meta = NULL;
if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", reg_arg_name(env, size_argno)); return -EACCES; }
if (reg_umin(size_reg) == 0 && !zero_size_allowed) { verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); return -EACCES; }
if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", reg_arg_name(env, size_argno)); return -EACCES; } err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), access_type, zero_size_allowed, meta); if (!err) { int regno = reg_from_argno(size_argno);
if (regno >= 0) err = mark_chain_precision(env, regno); else err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); } return err;}
static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size){ bool may_be_null = type_may_be_null(reg->type); struct bpf_reg_state saved_reg; int err;
if (bpf_register_is_null(reg)) return 0;
if (mem_size > S32_MAX) { verbose(env, "%s memory size %u is too large\n", reg_arg_name(env, argno), mem_size); return -EACCES; }
/* Assuming that the register contains a value check if the memory * access is safe. Temporarily save and restore the register's state as * the conversion shouldn't be visible to a caller. */ if (may_be_null) { saved_reg = *reg; mark_ptr_not_null_reg(reg); }
int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size;
err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL);
if (may_be_null) *reg = saved_reg;
return err;}
static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno){ bool may_be_null = type_may_be_null(mem_reg->type); struct bpf_reg_state saved_reg; struct bpf_call_arg_meta meta; int err;
memset(&meta, 0, sizeof(meta));
if (may_be_null) { saved_reg = *mem_reg; mark_ptr_not_null_reg(mem_reg); }
err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta);
if (may_be_null) *mem_reg = saved_reg;
return err;}
enum { PROCESS_SPIN_LOCK = (1 << 0), PROCESS_RES_LOCK = (1 << 1), PROCESS_LOCK_IRQ = (1 << 2),};
/* Implementation details: * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. * Two bpf_map_lookups (even with the same key) will have different reg->id. * Two separate bpf_obj_new will also have different reg->id. * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier * clears reg->id after value_or_null->value transition, since the verifier only * cares about the range of access to valid map value pointer and doesn't care * about actual address of the map element. * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps * reg->id > 0 after value_or_null->value transition. By doing so * two bpf_map_lookups will be considered two different pointers that * point to different bpf_spin_locks. Likewise for pointers to allocated objects * returned from bpf_obj_new. * The verifier allows taking only one bpf_spin_lock at a time to avoid * dead-locks. * Since only one bpf_spin_lock is allowed the checks are simpler than * reg_is_refcounted() logic. The verifier needs to remember only * one spin_lock instead of array of acquired_refs. * env->cur_state->active_locks remembers which map value element or allocated * object got locked and clears it after bpf_spin_unlock. */static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int flags){ bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; struct bpf_verifier_state *cur = env->cur_state; bool is_const = tnum_is_const(reg->var_off); bool is_irq = flags & PROCESS_LOCK_IRQ; u64 val = reg->var_off.value; struct bpf_map *map = NULL; struct btf *btf = NULL; struct btf_record *rec; u32 spin_lock_off; int err;
if (!is_const) { verbose(env, "%s doesn't have constant offset. %s_lock has to be at the constant offset\n", reg_arg_name(env, argno), lock_str); return -EINVAL; } if (reg->type == PTR_TO_MAP_VALUE) { map = reg->map_ptr; if (!map->btf) { verbose(env, "map '%s' has to have BTF in order to use %s_lock\n", map->name, lock_str); return -EINVAL; } } else { btf = reg->btf; }
rec = reg_btf_record(reg); if (!btf_record_has_field(rec, is_res_lock ? BPF_RES_SPIN_LOCK : BPF_SPIN_LOCK)) { verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local", map ? map->name : "kptr", lock_str); return -EINVAL; } spin_lock_off = is_res_lock ? rec->res_spin_lock_off : rec->spin_lock_off; if (spin_lock_off != val) { verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n", val, lock_str, spin_lock_off); return -EINVAL; } if (is_lock) { void *ptr; int type;
if (map) ptr = map; else ptr = btf;
if (!is_res_lock && cur->active_locks) { if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) { verbose(env, "Locking two bpf_spin_locks are not allowed\n"); return -EINVAL; } } else if (is_res_lock && cur->active_locks) { if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) { verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); return -EINVAL; } }
if (is_res_lock && is_irq) type = REF_TYPE_RES_LOCK_IRQ; else if (is_res_lock) type = REF_TYPE_RES_LOCK; else type = REF_TYPE_LOCK; err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr); if (err < 0) { verbose(env, "Failed to acquire lock state\n"); return err; } } else { void *ptr; int type;
if (map) ptr = map; else ptr = btf;
if (!cur->active_locks) { verbose(env, "%s_unlock without taking a lock\n", lock_str); return -EINVAL; }
if (is_res_lock && is_irq) type = REF_TYPE_RES_LOCK_IRQ; else if (is_res_lock) type = REF_TYPE_RES_LOCK; else type = REF_TYPE_LOCK; if (!find_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) { verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } if (release_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; }
invalidate_non_owning_refs(env); } return 0;}
/* Check if @regno is a pointer to a specific field in a map value */static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, enum btf_field_type field_type, struct bpf_map_desc *map_desc){ bool is_const = tnum_is_const(reg->var_off); struct bpf_map *map = reg->map_ptr; u64 val = reg->var_off.value; const char *struct_name = btf_field_type_name(field_type); int field_off = -1;
if (!is_const) { verbose(env, "%s doesn't have constant offset. %s has to be at the constant offset\n", reg_arg_name(env, argno), struct_name); return -EINVAL; } if (!map->btf) { verbose(env, "map '%s' has to have BTF in order to use %s\n", map->name, struct_name); return -EINVAL; } if (!btf_record_has_field(map->record, field_type)) { verbose(env, "map '%s' has no valid %s\n", map->name, struct_name); return -EINVAL; } switch (field_type) { case BPF_TIMER: field_off = map->record->timer_off; break; case BPF_TASK_WORK: field_off = map->record->task_work_off; break; case BPF_WORKQUEUE: field_off = map->record->wq_off; break; default: verifier_bug(env, "unsupported BTF field type: %s\n", struct_name); return -EINVAL; } if (field_off != val) { verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n", val, struct_name, field_off); return -EINVAL; } if (map_desc->ptr) { verifier_bug(env, "Two map pointers in a %s helper", struct_name); return -EFAULT; } map_desc->uid = reg->map_uid; map_desc->ptr = map; return 0;}
static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_map_desc *map){ if (IS_ENABLED(CONFIG_PREEMPT_RT)) { verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n"); return -EOPNOTSUPP; } return check_map_field_pointer(env, reg, argno, BPF_TIMER, map);}
static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_call_arg_meta *meta){ return process_timer_func(env, reg, argno, &meta->map);}
static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta){ return process_timer_func(env, reg, argno, &meta->map);}
static int process_kptr_func(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta){ struct bpf_reg_state *reg = reg_state(env, regno); struct btf_field *kptr_field; struct bpf_map *map_ptr; struct btf_record *rec; u32 kptr_off;
if (type_is_ptr_alloc_obj(reg->type)) { rec = reg_btf_record(reg); } else { /* PTR_TO_MAP_VALUE */ map_ptr = reg->map_ptr; if (!map_ptr->btf) { verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", map_ptr->name); return -EINVAL; } rec = map_ptr->record; meta->map.ptr = map_ptr; }
if (!tnum_is_const(reg->var_off)) { verbose(env, "R%d doesn't have constant offset. kptr has to be at the constant offset\n", regno); return -EINVAL; }
if (!btf_record_has_field(rec, BPF_KPTR)) { verbose(env, "R%d has no valid kptr\n", regno); return -EINVAL; }
kptr_off = reg->var_off.value; kptr_field = btf_record_find(rec, kptr_off, BPF_KPTR); if (!kptr_field) { verbose(env, "off=%d doesn't point to kptr\n", kptr_off); return -EACCES; } if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) { verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); return -EACCES; } meta->kptr_field = kptr_field; return 0;}
/* * Validate dynptr arguments for helper, kfunc and subprog. * * @dynptr is both input and output. It is populated when the argument is * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed) * and consumed when the argument is expecting to be an initialized dynptr. * @parent_id is used to track the referenced parent object (e.g., file or skb in * qdisc program) when constructing a dynptr. * * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. * * In both cases we deal with the first 8 bytes, but need to mark the next 8 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object. * * Mutability of bpf_dynptr is at two levels: the dynptr and the memory the * dynptr points to. At the first level, the verifier will make sure a * CONST_PTR_TO_DYNPTR cannot be reinitialized or destroyed. The mutability of * a dynptr's view (i.e., start and offset) is not tracked as there is not such * use case. The second level is tracked using the upper bit of bpf_dynptr->size * and checked dynamically during runtime. */static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, enum bpf_arg_type arg_type, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr){ int spi, err = 0;
if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { verbose(env, "%s expected pointer to stack or const struct bpf_dynptr\n", reg_arg_name(env, argno)); return -EINVAL; }
/* MEM_UNINIT - Points to memory that is an appropriate candidate for * constructing a mutable bpf_dynptr object. * * Currently, this is only possible with PTR_TO_STACK * pointing to a region of at least 16 bytes which doesn't * contain an existing bpf_dynptr. * * OBJ_RELEASE - Points to a initialized bpf_dynptr that will be * destroyed. * * None - Points to a initialized dynptr that cannot be * reinitialized or destroyed. However, the view of the * dynptr and the memory it points to may be mutated. */ if (arg_type & MEM_UNINIT) { int i;
if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); return -EINVAL; }
/* we write BPF_DW bits (8 bytes) at a time */ for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { err = check_mem_access(env, insn_idx, reg, argno, i, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; }
err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); return -EINVAL; }
if (!is_dynptr_reg_valid_init(env, reg)) { verbose(env, "Expected an initialized dynptr as %s\n", reg_arg_name(env, argno)); return -EINVAL; }
/* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { verbose(env, "Expected a dynptr of type %s as %s\n", dynptr_type_str(arg_to_dynptr_type(arg_type)), reg_arg_name(env, argno)); return -EINVAL; }
if (reg->type != CONST_PTR_TO_DYNPTR) { struct bpf_func_state *state = bpf_func(env, reg);
spi = dynptr_get_spi(env, reg); if (spi < 0) return spi;
/* * For CONST_PTR_TO_DYNPTR, reg is already scratched by check_reg_arg * in check_helper_call and mark_btf_func_reg_size in check_kfunc_call. */ mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS);
reg = &state->stack[spi].spilled_ptr; }
if (dynptr) { dynptr->type = reg->dynptr.type; dynptr->id = reg->id; dynptr->parent_id = reg->parent_id; } } return err;}
static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);}
static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_ITER_NEW;}
static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_ITER_DESTROY;}
static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, const struct btf_param *arg){ /* btf_check_iter_kfuncs() guarantees that first argument of any iter * kfunc is iter state pointer */ if (is_iter_kfunc(meta)) return arg_idx == 0;
/* iter passed as an argument to a generic kfunc */ return btf_param_match_suffix(meta->btf, arg, "__iter");}
static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, struct bpf_kfunc_call_arg_meta *meta){ struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id;
if (reg->type != PTR_TO_STACK) { verbose(env, "%s expected pointer to an iterator on stack\n", reg_arg_name(env, argno)); return -EINVAL; }
/* For iter_{new,next,destroy} functions, btf_check_iter_kfuncs() * ensures struct convention, so we wouldn't need to do any BTF * validation here. But given iter state can be passed as a parameter * to any kfunc, if arg has "__iter" suffix, we need to be a bit more * conservative here. */ btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx); if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); nr_slots = t->size / BPF_REG_SIZE;
if (is_iter_new_kfunc(meta)) { /* bpf_iter_<type>_new() expects pointer to uninit iter state */ if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { verbose(env, "expected uninitialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); return -EINVAL; }
for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) { err = check_mem_access(env, insn_idx, reg, argno, i, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err; }
err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots); if (err) return err; } else { /* iter_next() or iter_destroy(), as well as any kfunc * accepting iter argument, expect initialized iter state */ err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots); switch (err) { case 0: break; case -EINVAL: verbose(env, "expected an initialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); return err; default: return err; }
spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) return spi;
mark_stack_slots_scratched(env, spi, nr_slots);
/* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr);
if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); if (err) return err; } }
return 0;}
/* Look for a previous loop entry at insn_idx: nearest parent state * stopped at insn_idx with callsites matching those in cur->frame. */static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, int insn_idx){ struct bpf_verifier_state_list *sl; struct bpf_verifier_state *st; struct list_head *pos, *head;
/* Explored states are pushed in stack order, most recent states come first */ head = bpf_explored_state(env, insn_idx); list_for_each(pos, head) { sl = container_of(pos, struct bpf_verifier_state_list, node); /* If st->branches != 0 state is a part of current DFS verification path, * hence cur & st for a loop. */ st = &sl->state; if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) && st->dfs_depth < cur->dfs_depth) return st; }
return NULL;}
/* * Check if scalar registers are exact for the purpose of not widening. * More lenient than regs_exact() */static bool scalars_exact_for_widen(const struct bpf_reg_state *rold, const struct bpf_reg_state *rcur){ return !memcmp(rold, rcur, offsetof(struct bpf_reg_state, id));}
static void maybe_widen_reg(struct bpf_verifier_env *env, struct bpf_reg_state *rold, struct bpf_reg_state *rcur){ if (rold->type != SCALAR_VALUE) return; if (rold->type != rcur->type) return; if (rold->precise || rcur->precise || scalars_exact_for_widen(rold, rcur)) return; __mark_reg_unknown(env, rcur);}
static int widen_imprecise_scalars(struct bpf_verifier_env *env, struct bpf_verifier_state *old, struct bpf_verifier_state *cur){ struct bpf_func_state *fold, *fcur; int i, fr, num_slots;
for (fr = old->curframe; fr >= 0; fr--) { fold = old->frame[fr]; fcur = cur->frame[fr];
for (i = 0; i < MAX_BPF_REG; i++) maybe_widen_reg(env, &fold->regs[i], &fcur->regs[i]);
num_slots = min(fold->allocated_stack / BPF_REG_SIZE, fcur->allocated_stack / BPF_REG_SIZE); for (i = 0; i < num_slots; i++) { if (!bpf_is_spilled_reg(&fold->stack[i]) || !bpf_is_spilled_reg(&fcur->stack[i])) continue;
maybe_widen_reg(env, &fold->stack[i].spilled_ptr, &fcur->stack[i].spilled_ptr); } } return 0;}
static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st, struct bpf_kfunc_call_arg_meta *meta){ int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi;
return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr;}
/* process_iter_next_call() is called when verifier gets to iterator's next * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer * to it as just "iter_next()" in comments below. * * BPF verifier relies on a crucial contract for any iter_next() * implementation: it should *eventually* return NULL, and once that happens * it should keep returning NULL. That is, once iterator exhausts elements to * iterate, it should never reset or spuriously return new elements. * * With the assumption of such contract, process_iter_next_call() simulates * a fork in the verifier state to validate loop logic correctness and safety * without having to simulate infinite amount of iterations. * * In current state, we first assume that iter_next() returned NULL and * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such * conditions we should not form an infinite loop and should eventually reach * exit. * * Besides that, we also fork current state and enqueue it for later * verification. In a forked state we keep iterator state as ACTIVE * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We * also bump iteration depth to prevent erroneous infinite loop detection * later on (see iter_active_depths_differ() comment for details). In this * state we assume that we'll eventually loop back to another iter_next() * calls (it could be in exactly same location or in some other instruction, * it doesn't matter, we don't make any unnecessary assumptions about this, * everything revolves around iterator state in a stack slot, not which * instruction is calling iter_next()). When that happens, we either will come * to iter_next() with equivalent state and can conclude that next iteration * will proceed in exactly the same way as we just verified, so it's safe to * assume that loop converges. If not, we'll go on another iteration * simulation with a different input state, until all possible starting states * are validated or we reach maximum number of instructions limit. * * This way, we will either exhaustively discover all possible input states * that iterator loop can start with and eventually will converge, or we'll * effectively regress into bounded loop simulation logic and either reach * maximum number of instructions if loop is not provably convergent, or there * is some statically known limit on number of iterations (e.g., if there is * an explicit `if n > 100 then break;` statement somewhere in the loop). * * Iteration convergence logic in is_state_visited() relies on exact * states comparison, which ignores read and precision marks. * This is necessary because read and precision marks are not finalized * while in the loop. Exact comparison might preclude convergence for * simple programs like below: * * i = 0; * while(iter_next(&it)) * i++; * * At each iteration step i++ would produce a new distinct state and * eventually instruction processing limit would be reached. * * To avoid such behavior speculatively forget (widen) range for * imprecise scalar registers, if those registers were not precise at the * end of the previous iteration and do not match exactly. * * This is a conservative heuristic that allows to verify wide range of programs, * however it precludes verification of programs that conjure an * imprecise value on the first loop iteration and use it as precise on a second. * For example, the following safe program would fail to verify: * * struct bpf_num_iter it; * int arr[10]; * int i = 0, a = 0; * bpf_iter_num_new(&it, 0, 10); * while (bpf_iter_num_next(&it)) { * if (a == 0) { * a = 1; * i = 7; // Because i changed verifier would forget * // it's range on second loop entry. * } else { * arr[i] = 42; // This would fail to verify. * } * } * bpf_iter_num_destroy(&it); */static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, struct bpf_kfunc_call_arg_meta *meta){ struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; struct bpf_reg_state *cur_iter, *queued_iter;
BTF_TYPE_EMIT(struct bpf_iter);
cur_iter = get_iter_from_state(cur_st, meta);
if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE && cur_iter->iter.state != BPF_ITER_STATE_DRAINED) { verifier_bug(env, "unexpected iterator state %d (%s)", cur_iter->iter.state, iter_state_str(cur_iter->iter.state)); return -EFAULT; }
if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) { /* Because iter_next() call is a checkpoint is_state_visitied() * should guarantee parent state with same call sites and insn_idx. */ if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx || !same_callsites(cur_st->parent, cur_st)) { verifier_bug(env, "bad parent state for iter next call"); return -EFAULT; } /* Note cur_st->parent in the call below, it is necessary to skip * checkpoint created for cur_st by is_state_visited() * right at this instruction. */ prev_st = find_prev_entry(env, cur_st->parent, insn_idx); /* branch out active iter state */ queued_st = push_stack(env, insn_idx + 1, insn_idx, false); if (IS_ERR(queued_st)) return PTR_ERR(queued_st);
queued_iter = get_iter_from_state(queued_st, meta); queued_iter->iter.state = BPF_ITER_STATE_ACTIVE; queued_iter->iter.depth++; if (prev_st) widen_imprecise_scalars(env, prev_st, queued_st);
queued_fr = queued_st->frame[queued_st->curframe]; mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]); }
/* switch to DRAINED state, but keep the depth unchanged */ /* mark current iter state as drained and assume returned NULL */ cur_iter->iter.state = BPF_ITER_STATE_DRAINED; __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]);
return 0;}
static bool arg_type_is_mem_size(enum bpf_arg_type type){ return type == ARG_CONST_SIZE || type == ARG_CONST_SIZE_OR_ZERO;}
static bool arg_type_is_raw_mem(enum bpf_arg_type type){ return base_type(type) == ARG_PTR_TO_MEM && type & MEM_UNINIT;}
static bool arg_type_is_release(enum bpf_arg_type type){ return type & OBJ_RELEASE;}
static bool arg_type_is_dynptr(enum bpf_arg_type type){ return base_type(type) == ARG_PTR_TO_DYNPTR;}
static int resolve_map_arg_type(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type){ if (!meta->map.ptr) { /* kernel subsystem misconfigured verifier */ verifier_bug(env, "invalid map_ptr to access map->type"); return -EFAULT; }
switch (meta->map.ptr->map_type) { case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: if (*arg_type == ARG_PTR_TO_MAP_VALUE) { *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; } else { verbose(env, "invalid arg_type for sockmap/sockhash\n"); return -EINVAL; } break; case BPF_MAP_TYPE_BLOOM_FILTER: if (meta->func_id == BPF_FUNC_map_peek_elem) *arg_type = ARG_PTR_TO_MAP_VALUE; break; default: break; } return 0;}
struct bpf_reg_types { const enum bpf_reg_type types[10]; u32 *btf_id;};
static const struct bpf_reg_types sock_types = { .types = { PTR_TO_SOCK_COMMON, PTR_TO_SOCKET, PTR_TO_TCP_SOCK, PTR_TO_XDP_SOCK, },};
#ifdef CONFIG_NETstatic const struct bpf_reg_types btf_id_sock_common_types = { .types = { PTR_TO_SOCK_COMMON, PTR_TO_SOCKET, PTR_TO_TCP_SOCK, PTR_TO_XDP_SOCK, PTR_TO_BTF_ID, PTR_TO_BTF_ID | PTR_TRUSTED, }, .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],};#endif
static const struct bpf_reg_types mem_types = { .types = { PTR_TO_STACK, PTR_TO_PACKET, PTR_TO_PACKET_META, PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE, PTR_TO_MEM, PTR_TO_MEM | MEM_RINGBUF, PTR_TO_BUF, PTR_TO_BTF_ID | PTR_TRUSTED, PTR_TO_CTX, },};
static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, }};
static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } };static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID, PTR_TO_BTF_ID | PTR_TRUSTED, PTR_TO_BTF_ID | MEM_RCU, },};static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_BTF_ID | MEM_PERCPU, PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU, PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, }};static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };static const struct bpf_reg_types kptr_xchg_dest_types = { .types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, }};static const struct bpf_reg_types dynptr_types = { .types = { PTR_TO_STACK, CONST_PTR_TO_DYNPTR, }};
static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &mem_types, [ARG_PTR_TO_MAP_VALUE] = &mem_types, [ARG_CONST_SIZE] = &scalar_types, [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types,#ifdef CONFIG_NET [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,#endif [ARG_PTR_TO_SOCKET] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, [ARG_PTR_TO_MEM] = &mem_types, [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, [ARG_PTR_TO_STACK] = &stack_ptr_types, [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, [ARG_PTR_TO_TIMER] = &timer_types, [ARG_KPTR_XCHG_DEST] = &kptr_xchg_dest_types, [ARG_PTR_TO_DYNPTR] = &dynptr_types,};
static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type, const u32 *arg_btf_id, struct bpf_call_arg_meta *meta){ enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; int i, j, err;
compatible = compatible_reg_types[base_type(arg_type)]; if (!compatible) { verifier_bug(env, "unsupported arg type %d", arg_type); return -EFAULT; }
/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY * * Same for MAYBE_NULL: * * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL * * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type. * * Therefore we fold these flags depending on the arg_type before comparison. */ if (arg_type & MEM_RDONLY) type &= ~MEM_RDONLY; if (arg_type & PTR_MAYBE_NULL) type &= ~PTR_MAYBE_NULL; if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK;
/* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) { type &= ~MEM_ALLOC; type &= ~MEM_PERCPU; }
for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { expected = compatible->types[i]; if (expected == NOT_INIT) break;
if (type == expected) goto found; }
verbose(env, "%s type=%s expected=", reg_arg_name(env, argno), reg_type_str(env, reg->type)); for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); return -EACCES;
found: if (base_type(reg->type) != PTR_TO_BTF_ID) return 0;
if (compatible == &mem_types) { if (!(arg_type & MEM_RDONLY)) { verbose(env, "%s() may write into memory pointed by %s type=%s\n", func_id_name(meta->func_id), reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EACCES; } return 0; }
switch ((int)reg->type) { case PTR_TO_BTF_ID: case PTR_TO_BTF_ID | PTR_TRUSTED: case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: case PTR_TO_BTF_ID | MEM_RCU: case PTR_TO_BTF_ID | PTR_MAYBE_NULL: case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: { /* For bpf_sk_release, it needs to match against first member * 'struct sock_common', hence make an exception for it. This * allows bpf_sk_release to work for multiple socket types. */ bool strict_type_match = arg_type_is_release(arg_type) && meta->func_id != BPF_FUNC_sk_release;
if (type_may_be_null(reg->type) && (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { verbose(env, "Possibly NULL pointer passed to helper %s\n", reg_arg_name(env, argno)); return -EACCES; }
if (!arg_btf_id) { if (!compatible->btf_id) { verifier_bug(env, "missing arg compatible BTF ID"); return -EFAULT; } arg_btf_id = compatible->btf_id; }
if (meta->func_id == BPF_FUNC_kptr_xchg) { if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) return -EACCES; } else { if (arg_btf_id == BPF_PTR_POISON) { verbose(env, "verifier internal error:"); verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", reg_arg_name(env, argno)); return -EACCES; }
err = __check_ptr_off_reg(env, reg, argno, true); if (err) return err;
if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, strict_type_match)) { verbose(env, "%s is of type %s but %s is expected\n", reg_arg_name(env, argno), btf_type_name(reg->btf, reg->btf_id), btf_type_name(btf_vmlinux, *arg_btf_id)); return -EACCES; } } break; } case PTR_TO_BTF_ID | MEM_ALLOC: case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && meta->func_id != BPF_FUNC_kptr_xchg) { verifier_bug(env, "unimplemented handling of MEM_ALLOC"); return -EFAULT; } /* Check if local kptr in src arg matches kptr in dst arg */ if (meta->func_id == BPF_FUNC_kptr_xchg) { int regno = reg_from_argno(argno);
if (regno == BPF_REG_2 && map_kptr_match_type(env, meta->kptr_field, reg, regno)) return -EACCES; } break; case PTR_TO_BTF_ID | MEM_PERCPU: case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: /* Handled by helper specific checks */ break; default: verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match"); return -EFAULT; } return 0;}
static struct btf_field *reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields){ struct btf_field *field; struct btf_record *rec;
rec = reg_btf_record(reg); if (!rec) return NULL;
field = btf_record_find(rec, off, fields); if (!field) return NULL;
return field;}
static int __check_func_arg_reg_off(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type, bool btf_id_fixed_off_ok){ u32 type = reg->type;
/* When referenced register is passed to release function, its fixed * offset must be 0. * * We will check arg_type_is_release reg has id when storing * meta->release_regno. */ if (arg_type_is_release(arg_type)) { /* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it * may not directly point to the object being released, but to * dynptr pointing to such object, which might be at some offset * on the stack. In that case, we simply to fallback to the * default handling. */ if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK) return 0;
/* Doing check_ptr_off_reg check for the offset will catch this * because fixed_off_ok is false, but checking here allows us * to give the user a better error message. */ if (!tnum_is_const(reg->var_off) || reg->var_off.value != 0) { verbose(env, "%s must have zero offset when passed to release func or trusted arg to kfunc\n", reg_arg_name(env, argno)); return -EINVAL; } }
switch (type) { /* Pointer types where both fixed and variable offset is explicitly allowed: */ case PTR_TO_STACK: case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_MAP_KEY: case PTR_TO_MAP_VALUE: case PTR_TO_MEM: case PTR_TO_MEM | MEM_RDONLY: case PTR_TO_MEM | MEM_RINGBUF: case PTR_TO_BUF: case PTR_TO_BUF | MEM_RDONLY: case PTR_TO_ARENA: case SCALAR_VALUE: return 0; /* All the rest must be rejected, except PTR_TO_BTF_ID which allows * fixed offset. */ case PTR_TO_BTF_ID: case PTR_TO_BTF_ID | MEM_ALLOC: case PTR_TO_BTF_ID | PTR_TRUSTED: case PTR_TO_BTF_ID | MEM_RCU: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, * its fixed offset must be 0. In the other cases, fixed offset * can be non-zero unless the caller requires otherwise. * var_off always must be 0 for PTR_TO_BTF_ID, hence we still * need to do checks instead of returning. */ return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but * only when the argument is passed as memory, not ctx, * otherwise we may get modified ctx in tail called programs and * global subprogs (that may act as extension prog hooks). */ if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) return 0; fallthrough; default: return __check_ptr_off_reg(env, reg, argno, false); }}
static int check_func_arg_reg_off(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, enum bpf_arg_type arg_type){ return __check_func_arg_reg_off(env, reg, argno, arg_type, true);}
static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno){ struct bpf_map *map = reg->map_ptr; int err; int map_off; u64 map_addr; char *str_ptr;
if (reg->type != PTR_TO_MAP_VALUE) return -EINVAL;
if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { verbose(env, "%s points to insn_array map which cannot be used as const string\n", reg_arg_name(env, argno)); return -EACCES; }
if (!bpf_map_is_rdonly(map)) { verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); return -EACCES; }
if (!tnum_is_const(reg->var_off)) { verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno)); return -EACCES; }
if (!map->ops->map_direct_value_addr) { verbose(env, "no direct value access support for this map type\n"); return -EACCES; }
err = check_map_access(env, reg, argno, 0, map->value_size - reg->var_off.value, false, ACCESS_HELPER); if (err) return err;
map_off = reg->var_off.value; err = map->ops->map_direct_value_addr(map, &map_addr, map_off); if (err) { verbose(env, "direct value access on string failed\n"); return err; }
str_ptr = (char *)(long)(map_addr); if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { verbose(env, "string is not zero-terminated\n"); return -EINVAL; } return 0;}
/* Returns constant key value in `value` if possible, else negative error */static int get_constant_map_key(struct bpf_verifier_env *env, struct bpf_reg_state *key, u32 key_size, s64 *value){ struct bpf_func_state *state = bpf_func(env, key); struct bpf_reg_state *reg; int slot, spi, off; int spill_size = 0; int zero_size = 0; int stack_off; int i, err; u8 *stype;
if (!env->bpf_capable) return -EOPNOTSUPP; if (key->type != PTR_TO_STACK) return -EOPNOTSUPP; if (!tnum_is_const(key->var_off)) return -EOPNOTSUPP;
stack_off = key->var_off.value; slot = -stack_off - 1; spi = slot / BPF_REG_SIZE; off = slot % BPF_REG_SIZE; stype = state->stack[spi].slot_type;
/* First handle precisely tracked STACK_ZERO */ for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) zero_size++; if (zero_size >= key_size) { *value = 0; return 0; }
/* Check that stack contains a scalar spill of expected size */ if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) return -EOPNOTSUPP; for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) spill_size++; if (spill_size != key_size) return -EOPNOTSUPP;
reg = &state->stack[spi].spilled_ptr; if (!tnum_is_const(reg->var_off)) /* Stack value not statically known */ return -EOPNOTSUPP;
/* We are relying on a constant value. So mark as precise * to prevent pruning on it. */ bpf_bt_set_frame_slot(&env->bt, key->frameno, spi); err = mark_chain_precision_batch(env, env->cur_state); if (err < 0) return err;
*value = reg->var_off.value; return 0;}
static bool can_elide_value_nullness(const struct bpf_map *map);
static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, const struct bpf_func_proto *fn, int insn_idx){ u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; argno_t argno = argno_from_arg(arg + 1); enum bpf_reg_type type = reg->type; u32 *arg_btf_id = NULL; u32 key_size; int err = 0;
if (arg_type == ARG_DONTCARE) return 0;
err = check_reg_arg(env, regno, SRC_OP); if (err) return err;
if (arg_type == ARG_ANYTHING) { if (is_pointer_value(env, regno)) { verbose(env, "R%d leaks addr into helper function\n", regno); return -EACCES; } return 0; }
if (type_is_pkt_pointer(type) && !may_access_direct_pkt_data(env, meta, BPF_READ)) { verbose(env, "helper access to the packet is not allowed\n"); return -EACCES; }
if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { err = resolve_map_arg_type(env, meta, &arg_type); if (err) return err; }
if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) /* A NULL register has a SCALAR_VALUE type, so skip * type checking. */ goto skip_type_check;
/* arg_btf_id and arg_size are in a union. */ if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg];
err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta); if (err) return err;
err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type); if (err) return err;
skip_type_check: if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_id_name(meta->func_id), reg_arg_name(env, argno)); return -EINVAL; }
if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg);
switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ if (meta->map.ptr) { /* Use map_uid (which is unique id of inner map) to reject: * inner_map1 = bpf_map_lookup_elem(outer_map, key1) * inner_map2 = bpf_map_lookup_elem(outer_map, key2) * if (inner_map1 && inner_map2) { * timer = bpf_map_lookup_elem(inner_map1); * if (timer) * // mismatch would have been allowed * bpf_timer_init(timer, inner_map2); * } * * Comparing map_ptr is enough to distinguish normal and outer maps. */ if (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid) { verbose(env, "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", meta->map.uid, reg->map_uid); return -EINVAL; } } meta->map.ptr = reg->map_ptr; meta->map.uid = reg->map_uid; break; case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: * check that [key, key + map->key_size) are within * stack limits and initialized */ if (!meta->map.ptr) { /* in function declaration map_ptr must come before * map_key, so that it's verified and known before * we have to check map_key here. Otherwise it means * that kernel subsystem misconfigured verifier */ verifier_bug(env, "invalid map_ptr to access map->key"); return -EFAULT; } key_size = meta->map.ptr->key_size; err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr)) { err = get_constant_map_key(env, reg, key_size, &meta->const_map_key); if (err < 0) { meta->const_map_key = -1; if (err == -EOPNOTSUPP) err = 0; else return err; } } break; case ARG_PTR_TO_MAP_VALUE: if (type_may_be_null(arg_type) && bpf_register_is_null(reg)) return 0;
/* bpf_map_xxx(..., map_ptr, ..., value) call: * check [value, value + map->value_size) validity */ if (!meta->map.ptr) { /* kernel subsystem misconfigured verifier */ verifier_bug(env, "invalid map_ptr to access map->value"); return -EFAULT; } meta->raw_mode = arg_type & MEM_UNINIT; err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { verbose(env, "Helper has invalid btf_id in R%d\n", regno); return -EACCES; } meta->ret_btf = reg->btf; meta->ret_btf_id = reg->btf_id; break; case ARG_PTR_TO_SPIN_LOCK: if (in_rbtree_lock_required_cb(env)) { verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); return -EACCES; } if (meta->func_id == BPF_FUNC_spin_lock) { err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK); if (err) return err; } else if (meta->func_id == BPF_FUNC_spin_unlock) { err = process_spin_lock(env, reg, argno_from_reg(regno), 0); if (err) return err; } else { verifier_bug(env, "spin lock arg on unexpected helper"); return -EFAULT; } break; case ARG_PTR_TO_TIMER: err = process_timer_helper(env, reg, argno_from_reg(regno), meta); if (err) return err; break; case ARG_PTR_TO_FUNC: meta->subprogno = reg->subprogno; break; case ARG_PTR_TO_MEM: /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg], arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); if (err) return err; if (arg_type & MEM_ALIGNED) err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; case ARG_CONST_SIZE: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno_from_reg(regno), fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta); break; case ARG_CONST_SIZE_OR_ZERO: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno_from_reg(regno), fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, true, meta); break; case ARG_PTR_TO_DYNPTR: err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: if (!tnum_is_const(reg->var_off)) { verbose(env, "R%d is not a known constant'\n", regno); return -EACCES; } meta->mem_size = reg->var_off.value; err = mark_chain_precision(env, regno); if (err) return err; break; case ARG_PTR_TO_CONST_STR: { err = check_arg_const_str(env, reg, argno_from_reg(regno)); if (err) return err; break; } case ARG_KPTR_XCHG_DEST: err = process_kptr_func(env, regno, meta); if (err) return err; break; }
return err;}
static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id){ enum bpf_attach_type eatype = env->prog->expected_attach_type; enum bpf_prog_type type = resolve_prog_type(env->prog);
if (func_id != BPF_FUNC_map_update_elem && func_id != BPF_FUNC_map_delete_elem) return false;
/* It's not possible to get access to a locked struct sock in these * contexts, so updating is safe. */ switch (type) { case BPF_PROG_TYPE_TRACING: if (eatype == BPF_TRACE_ITER) return true; break; case BPF_PROG_TYPE_SOCK_OPS: /* map_update allowed only via dedicated helpers with event type checks */ if (func_id == BPF_FUNC_map_delete_elem) return true; break; case BPF_PROG_TYPE_SOCKET_FILTER: case BPF_PROG_TYPE_SCHED_CLS: case BPF_PROG_TYPE_SCHED_ACT: case BPF_PROG_TYPE_XDP: case BPF_PROG_TYPE_SK_REUSEPORT: case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_SK_LOOKUP: return true; default: break; }
verbose(env, "cannot update sockmap in this context\n"); return false;}
bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env){ return env->prog->jit_requested && bpf_jit_supports_subprog_tailcalls();}
static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func_id){ if (!map) return 0;
/* We need a two way check, first is from map perspective ... */ switch (map->map_type) { case BPF_MAP_TYPE_PROG_ARRAY: if (func_id != BPF_FUNC_tail_call) goto error; break; case BPF_MAP_TYPE_PERF_EVENT_ARRAY: if (func_id != BPF_FUNC_perf_event_read && func_id != BPF_FUNC_perf_event_output && func_id != BPF_FUNC_skb_output && func_id != BPF_FUNC_perf_event_read_value && func_id != BPF_FUNC_xdp_output) goto error; break; case BPF_MAP_TYPE_RINGBUF: if (func_id != BPF_FUNC_ringbuf_output && func_id != BPF_FUNC_ringbuf_reserve && func_id != BPF_FUNC_ringbuf_query && func_id != BPF_FUNC_ringbuf_reserve_dynptr && func_id != BPF_FUNC_ringbuf_submit_dynptr && func_id != BPF_FUNC_ringbuf_discard_dynptr) goto error; break; case BPF_MAP_TYPE_USER_RINGBUF: if (func_id != BPF_FUNC_user_ringbuf_drain) goto error; break; case BPF_MAP_TYPE_STACK_TRACE: if (func_id != BPF_FUNC_get_stackid) goto error; break; case BPF_MAP_TYPE_CGROUP_ARRAY: if (func_id != BPF_FUNC_skb_under_cgroup && func_id != BPF_FUNC_current_task_under_cgroup) goto error; break; case BPF_MAP_TYPE_CGROUP_STORAGE: case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: if (func_id != BPF_FUNC_get_local_storage) goto error; break; case BPF_MAP_TYPE_DEVMAP: case BPF_MAP_TYPE_DEVMAP_HASH: if (func_id != BPF_FUNC_redirect_map && func_id != BPF_FUNC_map_lookup_elem) goto error; break; /* Restrict bpf side of cpumap and xskmap, open when use-cases * appear. */ case BPF_MAP_TYPE_CPUMAP: if (func_id != BPF_FUNC_redirect_map) goto error; break; case BPF_MAP_TYPE_XSKMAP: if (func_id != BPF_FUNC_redirect_map && func_id != BPF_FUNC_map_lookup_elem) goto error; break; case BPF_MAP_TYPE_ARRAY_OF_MAPS: case BPF_MAP_TYPE_HASH_OF_MAPS: if (func_id != BPF_FUNC_map_lookup_elem) goto error; break; case BPF_MAP_TYPE_SOCKMAP: if (func_id != BPF_FUNC_sk_redirect_map && func_id != BPF_FUNC_sock_map_update && func_id != BPF_FUNC_msg_redirect_map && func_id != BPF_FUNC_sk_select_reuseport && func_id != BPF_FUNC_map_lookup_elem && !may_update_sockmap(env, func_id)) goto error; break; case BPF_MAP_TYPE_SOCKHASH: if (func_id != BPF_FUNC_sk_redirect_hash && func_id != BPF_FUNC_sock_hash_update && func_id != BPF_FUNC_msg_redirect_hash && func_id != BPF_FUNC_sk_select_reuseport && func_id != BPF_FUNC_map_lookup_elem && !may_update_sockmap(env, func_id)) goto error; break; case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: if (func_id != BPF_FUNC_sk_select_reuseport) goto error; break; case BPF_MAP_TYPE_QUEUE: case BPF_MAP_TYPE_STACK: if (func_id != BPF_FUNC_map_peek_elem && func_id != BPF_FUNC_map_pop_elem && func_id != BPF_FUNC_map_push_elem) goto error; break; case BPF_MAP_TYPE_SK_STORAGE: if (func_id != BPF_FUNC_sk_storage_get && func_id != BPF_FUNC_sk_storage_delete && func_id != BPF_FUNC_kptr_xchg) goto error; break; case BPF_MAP_TYPE_INODE_STORAGE: if (func_id != BPF_FUNC_inode_storage_get && func_id != BPF_FUNC_inode_storage_delete && func_id != BPF_FUNC_kptr_xchg) goto error; break; case BPF_MAP_TYPE_TASK_STORAGE: if (func_id != BPF_FUNC_task_storage_get && func_id != BPF_FUNC_task_storage_delete && func_id != BPF_FUNC_kptr_xchg) goto error; break; case BPF_MAP_TYPE_CGRP_STORAGE: if (func_id != BPF_FUNC_cgrp_storage_get && func_id != BPF_FUNC_cgrp_storage_delete && func_id != BPF_FUNC_kptr_xchg) goto error; break; case BPF_MAP_TYPE_BLOOM_FILTER: if (func_id != BPF_FUNC_map_peek_elem && func_id != BPF_FUNC_map_push_elem) goto error; break; case BPF_MAP_TYPE_INSN_ARRAY: goto error; default: break; }
/* ... and second from the function itself. */ switch (func_id) { case BPF_FUNC_tail_call: if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) goto error; if (env->subprog_cnt > 1 && !bpf_allow_tail_call_in_subprogs(env)) { verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n"); return -EINVAL; } break; case BPF_FUNC_perf_event_read: case BPF_FUNC_perf_event_output: case BPF_FUNC_perf_event_read_value: case BPF_FUNC_skb_output: case BPF_FUNC_xdp_output: if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) goto error; break; case BPF_FUNC_ringbuf_output: case BPF_FUNC_ringbuf_reserve: case BPF_FUNC_ringbuf_query: case BPF_FUNC_ringbuf_reserve_dynptr: case BPF_FUNC_ringbuf_submit_dynptr: case BPF_FUNC_ringbuf_discard_dynptr: if (map->map_type != BPF_MAP_TYPE_RINGBUF) goto error; break; case BPF_FUNC_user_ringbuf_drain: if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) goto error; break; case BPF_FUNC_get_stackid: if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) goto error; break; case BPF_FUNC_current_task_under_cgroup: case BPF_FUNC_skb_under_cgroup: if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) goto error; break; case BPF_FUNC_redirect_map: if (map->map_type != BPF_MAP_TYPE_DEVMAP && map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && map->map_type != BPF_MAP_TYPE_CPUMAP && map->map_type != BPF_MAP_TYPE_XSKMAP) goto error; break; case BPF_FUNC_sk_redirect_map: case BPF_FUNC_msg_redirect_map: case BPF_FUNC_sock_map_update: if (map->map_type != BPF_MAP_TYPE_SOCKMAP) goto error; break; case BPF_FUNC_sk_redirect_hash: case BPF_FUNC_msg_redirect_hash: case BPF_FUNC_sock_hash_update: if (map->map_type != BPF_MAP_TYPE_SOCKHASH) goto error; break; case BPF_FUNC_get_local_storage: if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) goto error; break; case BPF_FUNC_sk_select_reuseport: if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && map->map_type != BPF_MAP_TYPE_SOCKMAP && map->map_type != BPF_MAP_TYPE_SOCKHASH) goto error; break; case BPF_FUNC_map_pop_elem: if (map->map_type != BPF_MAP_TYPE_QUEUE && map->map_type != BPF_MAP_TYPE_STACK) goto error; break; case BPF_FUNC_map_peek_elem: case BPF_FUNC_map_push_elem: if (map->map_type != BPF_MAP_TYPE_QUEUE && map->map_type != BPF_MAP_TYPE_STACK && map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) goto error; break; case BPF_FUNC_map_lookup_percpu_elem: if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && map->map_type != BPF_MAP_TYPE_PERCPU_HASH && map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) goto error; break; case BPF_FUNC_sk_storage_get: case BPF_FUNC_sk_storage_delete: if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) goto error; break; case BPF_FUNC_inode_storage_get: case BPF_FUNC_inode_storage_delete: if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) goto error; break; case BPF_FUNC_task_storage_get: case BPF_FUNC_task_storage_delete: if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) goto error; break; case BPF_FUNC_cgrp_storage_get: case BPF_FUNC_cgrp_storage_delete: if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) goto error; break; default: break; }
return 0;error: verbose(env, "cannot pass map_type %d into func %s#%d\n", map->map_type, func_id_name(func_id), func_id); return -EINVAL;}
static bool check_raw_mode_ok(const struct bpf_func_proto *fn){ int count = 0;
if (arg_type_is_raw_mem(fn->arg1_type)) count++; if (arg_type_is_raw_mem(fn->arg2_type)) count++; if (arg_type_is_raw_mem(fn->arg3_type)) count++; if (arg_type_is_raw_mem(fn->arg4_type)) count++; if (arg_type_is_raw_mem(fn->arg5_type)) count++;
/* We only support one arg being in raw mode at the moment, * which is sufficient for the helper functions we have * right now. */ return count <= 1;}
static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg){ bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; bool has_size = fn->arg_size[arg] != 0; bool is_next_size = false;
if (arg + 1 < ARRAY_SIZE(fn->arg_type)) is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) return is_next_size;
return has_size == is_next_size || is_next_size == is_fixed;}
static bool check_arg_pair_ok(const struct bpf_func_proto *fn){ /* bpf_xxx(..., buf, len) call will access 'len' * bytes from memory 'buf'. Both arg types need * to be paired, so make sure there's no buggy * helper function specification. */ if (arg_type_is_mem_size(fn->arg1_type) || check_args_pair_invalid(fn, 0) || check_args_pair_invalid(fn, 1) || check_args_pair_invalid(fn, 2) || check_args_pair_invalid(fn, 3) || check_args_pair_invalid(fn, 4)) return false;
return true;}
static bool check_btf_id_ok(const struct bpf_func_proto *fn){ int i;
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) return fn->arg_btf_id[i] == BPF_PTR_POISON; if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && /* arg_btf_id and arg_size are in a union. */ (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || !(fn->arg_type[i] & MEM_FIXED_SIZE))) return false; }
return true;}
static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn){ int i;
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i];
if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) return false; }
return true;}
static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta){ int i;
for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i];
if (arg_type_is_release(arg_type)) { if (meta->release_regno) return false; meta->release_regno = i + 1; } }
return true;}
static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta){ return check_raw_mode_ok(fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && check_btf_id_ok(fn) ? 0 : -EINVAL;}
/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] * are now invalid, so turn them into unknown SCALAR_VALUE. * * This also applies to dynptr slices belonging to skb and xdp dynptrs, * since these slices point to packet data. */static void clear_all_pkt_pointers(struct bpf_verifier_env *env){ struct bpf_func_state *state; struct bpf_reg_state *reg;
bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) mark_reg_invalid(env, reg); }));}
enum { AT_PKT_END = -1, BEYOND_PKT_END = -2,};
static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open){ struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *reg = &state->regs[regn];
if (reg->type != PTR_TO_PACKET) /* PTR_TO_PACKET_META is not supported yet */ return;
/* The 'reg' is pkt > pkt_end or pkt >= pkt_end. * How far beyond pkt_end it goes is unknown. * if (!range_open) it's the case of pkt >= pkt_end * if (range_open) it's the case of pkt > pkt_end * hence this pointer is at least 1 byte bigger than pkt_end */ if (range_open) reg->range = BEYOND_PKT_END; else reg->range = AT_PKT_END;}
static int release_reference_nomark(struct bpf_verifier_state *state, int id){ int i;
for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].type != REF_TYPE_PTR) continue; if (state->refs[i].id == id) { release_reference_state(state, i); return 0; } } return -EINVAL;}
static int idstack_push(struct bpf_idmap *idmap, u32 id){ int i;
if (!id) return 0;
for (i = 0; i < idmap->cnt; i++) if (idmap->map[i].old == id) return 0;
if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE)) return -EFAULT;
idmap->map[idmap->cnt++].old = id; return 0;}
static int idstack_pop(struct bpf_idmap *idmap){ if (!idmap->cnt) return 0;
return idmap->map[--idmap->cnt].old;}
/* Release id and objects derived from it iteratively in a DFS manner */static int release_reference(struct bpf_verifier_env *env, int id){ u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); struct bpf_verifier_state *vstate = env->cur_state; struct bpf_idmap *idstack = &env->idmap_scratch; struct bpf_stack_state *stack; struct bpf_func_state *state; struct bpf_reg_state *reg; int i, err;
idstack->cnt = 0; err = idstack_push(idstack, id); if (err) return err;
if (find_reference_state(vstate, id)) WARN_ON_ONCE(release_reference_nomark(vstate, id));
while ((id = idstack_pop(idstack))) { /* * Child references are inaccessible after parent is released, * any child references that exist at this point are a leak. */ for (i = 0; i < vstate->acquired_refs; i++) { if (vstate->refs[i].type != REF_TYPE_PTR) continue; if (vstate->refs[i].parent_id != id) continue; verbose(env, "Leaking reference id=%d alloc_insn=%d. Release it first.\n", vstate->refs[i].id, vstate->refs[i].insn_idx); return -EINVAL; }
bpf_for_each_reg_in_vstate_mask(vstate, state, reg, stack, mask, ({ if (reg->id != id && reg->parent_id != id) continue;
/* Free objects derived from the current object */ if (reg->parent_id == id) { err = idstack_push(idstack, reg->id); if (err) return err; }
if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) mark_reg_invalid(env, reg); else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) invalidate_dynptr(env, stack); })); }
return 0;}
static void invalidate_non_owning_refs(struct bpf_verifier_env *env){ struct bpf_func_state *unused; struct bpf_reg_state *reg;
bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ if (type_is_non_owning_ref(reg->type)) mark_reg_invalid(env, reg); }));}
static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env){ struct bpf_stack_state *stack; struct bpf_func_state *state; struct bpf_reg_state *reg; u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; } }));}
static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id){ struct bpf_func_state *state; struct bpf_reg_state *reg; int err;
err = release_reference_nomark(env->cur_state, id);
bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg->id != id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; } }));
return err;}
static void clear_caller_saved_regs(struct bpf_verifier_env *env, struct bpf_reg_state *regs){ int i;
/* after the call registers r0 - r5 were scratched */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); __check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK); }}
static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env, struct bpf_func_state *state){ int i, nslots = state->out_stack_arg_cnt;
for (i = 0; i < nslots; i++) bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]);}
typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx);
static int set_callee_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx);
static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, set_callee_state_fn set_callee_state_cb, struct bpf_verifier_state *state){ struct bpf_func_state *caller, *callee; int err;
if (state->curframe + 1 >= MAX_CALL_FRAMES) { verbose(env, "the call stack of %d frames is too deep\n", state->curframe + 2); return -E2BIG; }
if (state->frame[state->curframe + 1]) { verifier_bug(env, "Frame %d already allocated", state->curframe + 1); return -EFAULT; }
caller = state->frame[state->curframe]; callee = kzalloc_obj(*callee, GFP_KERNEL_ACCOUNT); if (!callee) return -ENOMEM; state->frame[state->curframe + 1] = callee;
/* callee cannot access r0, r6 - r9 for reading and has to write * into its own stack before reading from it. * callee can read/write into caller's stack */ init_func_state(env, callee, /* remember the callsite, it will be used by bpf_exit */ callsite, state->curframe + 1 /* frameno within this callchain */, subprog /* subprog number within this prog */); err = set_callee_state_cb(env, caller, callee, callsite); if (err) goto err_out;
/* only increment it after check_reg_arg() finished */ state->curframe++;
return 0;
err_out: free_func_state(callee); state->frame[state->curframe + 1] = NULL; return err;}
static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, const struct btf *btf, struct bpf_reg_state *regs){ struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; struct ref_obj_desc ref_obj = {}; u32 i; int ret, err;
ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { err = check_outgoing_stack_args(env, caller, sub->arg_cnt); if (err) return err; } return ret; }
ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); if (ret) return ret;
/* check that BTF function arguments match actual types that the * verifier sees. */ for (i = 0; i < sub->arg_cnt; i++) { argno_t argno = argno_from_arg(i + 1); struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); struct bpf_subprog_arg_info *arg = &sub->args[i];
if (arg->arg_type == ARG_ANYTHING) { if (reg->type != SCALAR_VALUE) { bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); return -EINVAL; } } else if (arg->arg_type & PTR_UNTRUSTED) { /* * Anything is allowed for untrusted arguments, as these are * read-only and probe read instructions would protect against * invalid memory access. */ } else if (arg->arg_type == ARG_PTR_TO_CTX) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); if (ret < 0) return ret; /* If function expects ctx type in BTF check that caller * is passing PTR_TO_CTX. */ if (reg->type != PTR_TO_CTX) { bpf_log(log, "%s expects pointer to ctx\n", reg_arg_name(env, argno)); return -EINVAL; } } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; if (check_mem_reg(env, reg, argno, arg->mem_size)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { bpf_log(log, "%s is expected to be non-NULL\n", reg_arg_name(env, argno)); return -EINVAL; } } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { /* * Can pass any value and the kernel won't crash, but * only PTR_TO_ARENA or SCALAR make sense. Everything * else is a bug in the bpf program. Point it out to * the user at the verification time instead of * run-time debug nightmare. */ if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { bpf_log(log, "%s is not a pointer to arena or scalar.\n", reg_arg_name(env, argno)); return -EINVAL; } } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); if (ret) return ret;
ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { struct bpf_call_arg_meta meta; int err;
if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) continue;
memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; } else { verifier_bug(env, "unrecognized %s type %d", reg_arg_name(env, argno), arg->arg_type); return -EFAULT; } }
return 0;}
/* Compare BTF of a function call with given bpf_reg_state. * Returns: * EFAULT - there is a verifier bug. Abort verification. * EINVAL - there is a type mismatch or BTF is not available. * 0 - BTF matches with what bpf_reg_state expects. * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. */static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs){ struct bpf_prog *prog = env->prog; struct btf *btf = prog->aux->btf; u32 btf_id; int err;
if (!prog->aux->func_info) return -EINVAL;
btf_id = prog->aux->func_info[subprog].type_id; if (!btf_id) return -EFAULT;
if (prog->aux->func_info_aux[subprog].unreliable) return -EINVAL;
err = btf_check_func_arg_match(env, subprog, btf, regs); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. * In such cases mark the function as unreliable from BTF point of view. */ if (err) prog->aux->func_info_aux[subprog].unreliable = true; return err;}
static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, int subprog, set_callee_state_fn set_callee_state_cb){ struct bpf_verifier_state *state = env->cur_state, *callback_state; struct bpf_func_state *caller, *callee; int err;
caller = state->frame[state->curframe]; err = btf_check_subprog_call(env, subprog, caller->regs); if (err == -EFAULT) return err;
/* set_callee_state is used for direct subprog calls, but we are * interested in validating only BPF helpers that can call subprogs as * callbacks */ env->subprog_info[subprog].is_cb = true; if (bpf_pseudo_kfunc_call(insn) && !is_callback_calling_kfunc(insn->imm)) { verifier_bug(env, "kfunc %s#%d not marked as callback-calling", func_id_name(insn->imm), insn->imm); return -EFAULT; } else if (!bpf_pseudo_kfunc_call(insn) && !is_callback_calling_function(insn->imm)) { /* helper */ verifier_bug(env, "helper %s#%d not marked as callback-calling", func_id_name(insn->imm), insn->imm); return -EFAULT; }
if (bpf_is_async_callback_calling_insn(insn)) { struct bpf_verifier_state *async_cb;
/* there is no real recursion here. timer and workqueue callbacks are async */ env->subprog_info[subprog].is_async_cb = true; async_cb = push_async_cb(env, env->subprog_info[subprog].start, insn_idx, subprog, is_async_cb_sleepable(env, insn)); if (IS_ERR(async_cb)) return PTR_ERR(async_cb); callee = async_cb->frame[0]; callee->async_entry_cnt = caller->async_entry_cnt + 1;
/* Convert bpf_timer_set_callback() args into timer callback args */ err = set_callee_state_cb(env, caller, callee, insn_idx); if (err) return err;
return 0; }
/* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); if (IS_ERR(callback_state)) return PTR_ERR(callback_state);
err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, callback_state); if (err) return err;
callback_state->callback_unroll_depth++; callback_state->frame[callback_state->curframe - 1]->callback_depth++; caller->callback_depth = 0; return 0;}
static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit);
static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx){ struct bpf_verifier_state *state = env->cur_state; struct bpf_subprog_info *caller_info; u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; int err, subprog, target_insn;
target_insn = *insn_idx + insn->imm + 1; subprog = bpf_find_subprog(env, target_insn); if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", target_insn)) return -EFAULT;
caller = state->frame[state->curframe]; err = btf_check_subprog_call(env, subprog, caller->regs); if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { const char *sub_name = subprog_name(env, subprog);
if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" "use static function instead\n"); return -EINVAL; }
if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); return -EINVAL; }
if (err) { verbose(env, "Caller passes invalid args into func#%d ('%s')\n", subprog, sub_name); return err; }
if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); /* mark global subprog for verifying after main prog */ subprog_aux(env, subprog)->called = true; clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env));
/* All non-void global functions return a 64-bit SCALAR_VALUE. */ if (!subprog_returns_void(env, subprog)) { mark_reg_unknown(env, caller->regs, BPF_REG_0); caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; }
if (env->subprog_info[subprog].might_throw) { struct bpf_verifier_state *branch;
branch = push_stack(env, *insn_idx + 1, *insn_idx, false); if (IS_ERR(branch)) { verbose(env, "failed to push state for global subprog exception path\n"); return PTR_ERR(branch); } return process_bpf_exit_full(env, NULL, true); }
/* continue with next insn after call */ return 0; }
/* * Track caller's total stack arg count (incoming + max outgoing). * This is needed so the JIT knows how much stack arg space to allocate. */ caller_info = &env->subprog_info[caller->subprogno]; callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; if (stack_arg_cnt > caller_info->stack_arg_cnt) caller_info->stack_arg_cnt = stack_arg_cnt;
/* for regular function entry setup new frame and continue * from that frame. */ err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); if (err) return err;
clear_caller_saved_regs(env, caller->regs);
/* and go analyze first insn of the callee */ *insn_idx = env->subprog_info[subprog].start - 1;
if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "caller:\n"); print_verifier_state(env, state, caller->frameno, true); verbose(env, "callee:\n"); print_verifier_state(env, state, state->curframe, true); }
return 0;}
int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee){ /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, * void *callback_ctx, u64 flags); * callback_fn(struct bpf_map *map, void *key, void *value, * void *callback_ctx); */ callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
/* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
/* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); return 0;}
static int set_callee_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ int i;
/* copy r1 - r5 args that callee can access. The copy includes parent * pointers, which connects us up to the liveness chain */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) callee->regs[i] = caller->regs[i]; return 0;}
static int set_map_elem_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; struct bpf_map *map; int err;
/* valid map_ptr and poison value does not matter */ map = insn_aux->map_ptr_state.map_ptr; if (!map->ops->map_set_for_each_callback_args || !map->ops->map_for_each_callback) { verbose(env, "callback function not allowed for map\n"); return -ENOTSUPP; }
err = map->ops->map_set_for_each_callback_args(env, caller, callee); if (err) return err;
callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0;}
static int set_loop_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, * u64 flags); * callback_fn(u64 index, void *callback_ctx); */ callee->regs[BPF_REG_1].type = SCALAR_VALUE; callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
/* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0;}
static int set_timer_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); * callback_fn(struct bpf_map *map, void *key, void *value); */ callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr;
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr;
/* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_async_callback_fn = true; callee->callback_ret_range = retval_range(0, 0); return 0;}
static int set_find_vma_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ /* bpf_find_vma(struct task_struct *task, u64 addr, * void *callback_fn, void *callback_ctx, u64 flags) * (callback_fn)(struct task_struct *task, * struct vm_area_struct *vma, void *callback_ctx); */ callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].btf = btf_vmlinux; callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA];
/* pointer to stack or null */ callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];
/* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0;}
static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void * callback_ctx, u64 flags); * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx); */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_0]); mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
/* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0;}
static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ /* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, * bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)); * * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd * by this point, so look at 'root' */ struct btf_field *field;
field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].var_off.value, BPF_RB_ROOT); if (!field || !field->graph_root.value_btf_id) return -EFAULT;
mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root); ref_set_non_owning(env, &callee->regs[BPF_REG_1]); mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root); ref_set_non_owning(env, &callee->regs[BPF_REG_2]);
bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0;}
static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee, int insn_idx){ struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr;
/* * callback_fn(struct bpf_map *map, void *key, void *value); */ callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; __mark_reg_known_zero(&callee->regs[BPF_REG_1]); callee->regs[BPF_REG_1].map_ptr = map_ptr;
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; __mark_reg_known_zero(&callee->regs[BPF_REG_2]); callee->regs[BPF_REG_2].map_ptr = map_ptr;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr;
/* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_async_callback_fn = true; callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); return 0;}
static bool is_rbtree_lock_required_kfunc(u32 btf_id);
/* Are we currently verifying the callback for a rbtree helper that must * be called with lock held? If so, no need to complain about unreleased * lock */static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env){ struct bpf_verifier_state *state = env->cur_state; struct bpf_insn *insn = env->prog->insnsi; struct bpf_func_state *callee; int kfunc_btf_id;
if (!state->curframe) return false;
callee = state->frame[state->curframe];
if (!callee->in_callback_fn) return false;
kfunc_btf_id = insn[callee->callsite].imm; return is_rbtree_lock_required_kfunc(kfunc_btf_id);}
static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg){ if (range.return_32bit) return range.minval <= reg_s32_min(reg) && reg_s32_max(reg) <= range.maxval; else return range.minval <= reg_smin(reg) && reg_smax(reg) <= range.maxval;}
static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx){ struct bpf_verifier_state *state = env->cur_state, *prev_st; struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; int err;
callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; if (r0->type == PTR_TO_STACK) { /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack * pointer will be invalid as soon as function exits, * but let's be conservative */ verbose(env, "cannot return stack pointer to the caller\n"); return -EINVAL; }
caller = state->frame[state->curframe - 1]; if (callee->in_callback_fn) { if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); return -EACCES; }
/* we are going to rely on register's precise value */ err = mark_chain_precision(env, BPF_REG_0); if (err) return err;
/* enforce R0 return value range, and bpf_callback_t returns 64bit */ if (!retval_range_within(callee->callback_ret_range, r0)) { verbose_invalid_scalar(env, r0, callee->callback_ret_range, "At callback return", "R0"); return -EINVAL; } if (!bpf_calls_callback(env, callee->callsite)) { verifier_bug(env, "in callback at %d, callsite %d !calls_callback", *insn_idx, callee->callsite); return -EFAULT; } } else { /* return to the caller whatever r0 had in the callee */ caller->regs[BPF_REG_0] = *r0; }
/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, * there function call logic would reschedule callback visit. If iteration * converges is_state_visited() would prune that visit eventually. */ in_callback_fn = callee->in_callback_fn; if (in_callback_fn) *insn_idx = callee->callsite; else *insn_idx = callee->callsite + 1;
if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "returning from callee:\n"); print_verifier_state(env, state, callee->frameno, true); verbose(env, "to caller at %d:\n", *insn_idx); print_verifier_state(env, state, caller->frameno, true); } /* clear everything in the callee. In case of exceptional exits using * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); state->frame[state->curframe--] = NULL; invalidate_outgoing_stack_args(env, caller);
/* for callbacks widen imprecise scalars to make programs like below verify: * * struct ctx { int i; } * void cb(int idx, struct ctx *ctx) { ctx->i++; ... } * ... * struct ctx = { .i = 0; } * bpf_loop(100, cb, &ctx, 0); * * This is similar to what is done in process_iter_next_call() for open * coded iterators. */ prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL; if (prev_st) { err = widen_imprecise_scalars(env, prev_st, state); if (err) return err; } return 0;}
static int do_refine_retval_range(struct bpf_verifier_env *env, struct bpf_reg_state *regs, int ret_type, int func_id, struct bpf_call_arg_meta *meta){ struct bpf_retval_range range; struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
if (ret_type != RET_INTEGER) return 0;
switch (func_id) { case BPF_FUNC_get_stack: case BPF_FUNC_get_task_stack: case BPF_FUNC_probe_read_str: case BPF_FUNC_probe_read_kernel_str: case BPF_FUNC_probe_read_user_str: reg_set_srange64(ret_reg, -MAX_ERRNO, meta->msize_max_value); reg_set_srange32(ret_reg, -MAX_ERRNO, meta->msize_max_value); reg_bounds_sync(ret_reg); break; case BPF_FUNC_get_smp_processor_id: reg_set_urange64(ret_reg, 0, nr_cpu_ids - 1); reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1); reg_bounds_sync(ret_reg); break; case BPF_FUNC_get_retval: /* * bpf_get_retval may see arbitrary value passed by bpf_prog_run_array_cg for * CGROUP_GETSOCKOPT type. */ if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT && env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT) break;
if (prog_type == BPF_PROG_TYPE_LSM && env->prog->expected_attach_type == BPF_LSM_CGROUP) { if (!env->prog->aux->attach_func_proto->type) break; bpf_lsm_get_retval_range(env->prog, &range); } else { range.minval = -MAX_ERRNO; range.maxval = 0; }
reg_set_srange64(ret_reg, range.minval, range.maxval); reg_set_srange32(ret_reg, range.minval, range.maxval); reg_bounds_sync(ret_reg); break; }
return reg_bounds_sanity_check(env, ret_reg, "retval");}
static intrecord_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int func_id, int insn_idx){ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; struct bpf_map *map = meta->map.ptr;
if (func_id != BPF_FUNC_tail_call && func_id != BPF_FUNC_map_lookup_elem && func_id != BPF_FUNC_map_update_elem && func_id != BPF_FUNC_map_delete_elem && func_id != BPF_FUNC_map_push_elem && func_id != BPF_FUNC_map_pop_elem && func_id != BPF_FUNC_map_peek_elem && func_id != BPF_FUNC_for_each_map_elem && func_id != BPF_FUNC_redirect_map && func_id != BPF_FUNC_map_lookup_percpu_elem) return 0;
if (map == NULL) { verifier_bug(env, "expected map for helper call"); return -EFAULT; }
/* In case of read-only, some additional restrictions * need to be applied in order to prevent altering the * state of the map from program side. */ if ((map->map_flags & BPF_F_RDONLY_PROG) && (func_id == BPF_FUNC_map_delete_elem || func_id == BPF_FUNC_map_update_elem || func_id == BPF_FUNC_map_push_elem || func_id == BPF_FUNC_map_pop_elem)) { verbose(env, "write into map forbidden\n"); return -EACCES; }
if (!aux->map_ptr_state.map_ptr) bpf_map_ptr_store(aux, meta->map.ptr, !meta->map.ptr->bypass_spec_v1, false); else if (aux->map_ptr_state.map_ptr != meta->map.ptr) bpf_map_ptr_store(aux, meta->map.ptr, !meta->map.ptr->bypass_spec_v1, true); return 0;}
static intrecord_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int func_id, int insn_idx){ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; struct bpf_reg_state *reg; struct bpf_map *map = meta->map.ptr; u64 val, max; int err;
if (func_id != BPF_FUNC_tail_call) return 0; if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { verbose(env, "expected prog array map for tail call"); return -EINVAL; }
reg = reg_state(env, BPF_REG_3); val = reg->var_off.value; max = map->max_entries;
if (!(is_reg_const(reg, false) && val < max)) { bpf_map_key_store(aux, BPF_MAP_KEY_POISON); return 0; }
err = mark_chain_precision(env, BPF_REG_3); if (err) return err; if (bpf_map_key_unseen(aux)) bpf_map_key_store(aux, val); else if (!bpf_map_key_poisoned(aux) && bpf_map_key_immediate(aux) != val) bpf_map_key_store(aux, BPF_MAP_KEY_POISON); return 0;}
static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit){ struct bpf_verifier_state *state = env->cur_state; enum bpf_prog_type type = resolve_prog_type(env->prog); struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); bool refs_lingering = false; int i;
if (!exception_exit && cur_func(env)->frameno) return 0;
for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].type != REF_TYPE_PTR) continue; /* Allow struct_ops programs to return a referenced kptr back to * kernel. Type checks are performed later in check_return_code. */ if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit && reg->id == state->refs[i].id) continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); refs_lingering = true; } return refs_lingering ? -EINVAL : 0;}
static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix){ int err;
if (check_lock && env->cur_state->active_locks) { verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); return -EINVAL; }
err = check_reference_leak(env, exception_exit); if (err) { verbose(env, "%s would lead to reference leak\n", prefix); return err; }
if (check_lock && env->cur_state->active_irq_id) { verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); return -EINVAL; }
if (check_lock && env->cur_state->active_rcu_locks) { verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); return -EINVAL; }
if (check_lock && env->cur_state->active_preempt_locks) { verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); return -EINVAL; }
return 0;}
static int check_bpf_snprintf_call(struct bpf_verifier_env *env, struct bpf_reg_state *regs){ struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; struct bpf_map *fmt_map = fmt_reg->map_ptr; struct bpf_bprintf_data data = {}; int err, fmt_map_off, num_args; u64 fmt_addr; char *fmt;
/* data must be an array of u64 */ if (data_len_reg->var_off.value % 8) return -EINVAL; num_args = data_len_reg->var_off.value / 8;
/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const * and map_direct_value_addr is set. */ fmt_map_off = fmt_reg->var_off.value; err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, fmt_map_off); if (err) { verbose(env, "failed to retrieve map value address\n"); return -EFAULT; } fmt = (char *)(long)fmt_addr + fmt_map_off;
/* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we * can focus on validating the format specifiers. */ err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data); if (err < 0) verbose(env, "Invalid format string\n");
return err;}
static int check_get_func_ip(struct bpf_verifier_env *env){ enum bpf_prog_type type = resolve_prog_type(env->prog); int func_id = BPF_FUNC_get_func_ip;
if (type == BPF_PROG_TYPE_TRACING) { if (!bpf_prog_has_trampoline(env->prog)) { verbose(env, "func %s#%d supported only for fentry/fexit/fsession/fmod_ret programs\n", func_id_name(func_id), func_id); return -ENOTSUPP; } return 0; } else if (type == BPF_PROG_TYPE_KPROBE) { return 0; }
verbose(env, "func %s#%d not supported for program type %d\n", func_id_name(func_id), func_id, type); return -ENOTSUPP;}
static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env){ return &env->insn_aux_data[env->insn_idx];}
static bool loop_flag_is_zero(struct bpf_verifier_env *env){ struct bpf_reg_state *reg = reg_state(env, BPF_REG_4); bool reg_is_null = bpf_register_is_null(reg);
if (reg_is_null) mark_chain_precision(env, BPF_REG_4);
return reg_is_null;}
static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno){ struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
if (!state->initialized) { state->initialized = 1; state->fit_for_inline = loop_flag_is_zero(env); state->callback_subprogno = subprogno; return; }
if (!state->fit_for_inline) return;
state->fit_for_inline = (loop_flag_is_zero(env) && state->callback_subprogno == subprogno);}
/* Returns whether or not the given map can potentially elide * lookup return value nullness check. This is possible if the key * is statically known. */static bool can_elide_value_nullness(const struct bpf_map *map){ if (map->map_flags & BPF_F_INNER_MAP) return false;
switch (map->map_type) { case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_PERCPU_ARRAY: return true; default: return false; }}
int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, const struct bpf_func_proto **ptr){ if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) return -ERANGE;
if (!env->ops->get_func_proto) return -EINVAL;
*ptr = env->ops->get_func_proto(func_id, env->prog); return *ptr && (*ptr)->func ? 0 : -EINVAL;}
/* Check if we're in a sleepable context. */static inline bool in_sleepable_context(struct bpf_verifier_env *env){ return !env->cur_state->active_rcu_locks && !env->cur_state->active_preempt_locks && !env->cur_state->active_locks && !env->cur_state->active_irq_id && in_sleepable(env);}
static const char *non_sleepable_context_description(struct bpf_verifier_env *env){ if (env->cur_state->active_rcu_locks) return "rcu_read_lock region"; if (env->cur_state->active_preempt_locks) return "non-preemptible region"; if (env->cur_state->active_irq_id) return "IRQ-disabled region"; if (env->cur_state->active_locks) return "lock region"; return "non-sleepable prog";}
static int release_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, bool convert_rcu, bool release_dynptr){ int err = -EINVAL;
if (bpf_register_is_null(reg)) return 0;
if (release_dynptr) err = unmark_stack_slots_dynptr(env, reg); else if (convert_rcu) err = ref_convert_alloc_rcu_protected(env, reg->id); else if (reg_is_referenced(env, reg)) err = release_reference(env, reg->id);
return err;}
static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p){ enum bpf_prog_type prog_type = resolve_prog_type(env->prog); bool returns_cpu_specific_alloc_ptr = false; const struct bpf_func_proto *fn = NULL; enum bpf_return_type ret_type; enum bpf_type_flag ret_flag; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; int insn_idx = *insn_idx_p; bool changes_data; int i, err, func_id;
/* find function prototype */ func_id = insn->imm; err = bpf_get_helper_proto(env, insn->imm, &fn); if (err == -ERANGE) { verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id); return -EINVAL; }
if (err) { verbose(env, "program of this type cannot use helper %s#%d\n", func_id_name(func_id), func_id); return err; }
/* eBPF programs must be GPL compatible to use GPL-ed functions */ if (!env->prog->gpl_compatible && fn->gpl_only) { verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); return -EINVAL; }
if (fn->allowed && !fn->allowed(env->prog)) { verbose(env, "helper call is not allowed in probe\n"); return -EINVAL; }
/* With LD_ABS/IND some JITs save/restore skb from r1. */ changes_data = bpf_helper_changes_pkt_data(func_id); if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { verifier_bug(env, "func %s#%d: r1 != ctx", func_id_name(func_id), func_id); return -EFAULT; }
memset(&meta, 0, sizeof(meta)); meta.pkt_access = fn->pkt_access;
err = check_func_proto(fn, &meta); if (err) { verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); return err; }
if (fn->might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); return -EINVAL; }
/* Track non-sleepable context for helpers. */ if (!in_sleepable_context(env)) env->insn_aux_data[insn_idx].non_sleepable = true;
meta.func_id = func_id; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { err = check_func_arg(env, i, &meta, fn, insn_idx); if (err) return err; }
err = record_func_map(env, &meta, func_id, insn_idx); if (err) return err;
err = record_func_key(env, &meta, func_id, insn_idx); if (err) return err;
regs = cur_regs(env);
/* Mark slots with STACK_MISC in case of raw mode, stack offset * is inferred from register state. */ for (i = 0; i < meta.access_size; i++) { err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B, BPF_WRITE, -1, false, false); if (err) return err; }
if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; bool convert_rcu = (func_id == BPF_FUNC_kptr_xchg) && in_rcu_cs(env) && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU);
err = release_reg(env, reg, convert_rcu, !!meta.dynptr.id); if (err) return err; }
switch (func_id) { case BPF_FUNC_tail_call: err = check_resource_leak(env, false, true, "tail_call"); if (err) return err; break; case BPF_FUNC_get_local_storage: /* check that flags argument in get_local_storage(map, flags) is 0, * this is required because get_local_storage() can't return an error. */ if (!bpf_register_is_null(®s[BPF_REG_2])) { verbose(env, "get_local_storage() doesn't support non-zero flags\n"); return -EINVAL; } break; case BPF_FUNC_for_each_map_elem: err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_map_elem_callback_state); break; case BPF_FUNC_timer_set_callback: err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_timer_callback_state); break; case BPF_FUNC_find_vma: err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_find_vma_callback_state); break; case BPF_FUNC_snprintf: err = check_bpf_snprintf_call(env, regs); break; case BPF_FUNC_loop: update_loop_inline_state(env, meta.subprogno); /* Verifier relies on R1 value to determine if bpf_loop() iteration * is finished, thus mark it precise. */ err = mark_chain_precision(env, BPF_REG_1); if (err) return err; if (cur_func(env)->callback_depth < reg_umax(®s[BPF_REG_1])) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_loop_callback_state); } else { cur_func(env)->callback_depth = 0; if (env->log.level & BPF_LOG_LEVEL2) verbose(env, "frame%d bpf_loop iteration limit reached\n", env->cur_state->curframe); } break; case BPF_FUNC_dynptr_from_mem: if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", reg_type_str(env, regs[BPF_REG_1].type)); return -EACCES; } break; case BPF_FUNC_set_retval: { struct bpf_retval_range range = { .minval = -MAX_ERRNO, .maxval = 0, .return_32bit = true }; struct bpf_reg_state *r1 = ®s[BPF_REG_1];
if (r1->type != SCALAR_VALUE) { verbose(env, "R1 is not a scalar\n"); return -EINVAL; }
/* CGROUP_GETSOCKOPT is allowed to return arbitrary value */ if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT && env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT) break;
if (prog_type == BPF_PROG_TYPE_LSM && env->prog->expected_attach_type == BPF_LSM_CGROUP) { if (!env->prog->aux->attach_func_proto->type) { /* Make sure programs that attach to void * hooks don't try to modify return value. */ verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); return -EINVAL; } bpf_lsm_get_retval_range(env->prog, &range); }
err = mark_chain_precision(env, BPF_REG_1); if (err) return err;
if (!retval_range_within(range, r1)) { verbose_invalid_scalar(env, r1, range, "At bpf_set_retval", "R1"); return -EINVAL; }
break; } case BPF_FUNC_dynptr_write: { enum bpf_dynptr_type dynptr_type = meta.dynptr.type;
if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) return -EFAULT;
if (dynptr_type == BPF_DYNPTR_TYPE_SKB || dynptr_type == BPF_DYNPTR_TYPE_SKB_META) /* this will trigger clear_all_pkt_pointers(), which will * invalidate all dynptr slices associated with the skb */ changes_data = true;
break; } case BPF_FUNC_per_cpu_ptr: case BPF_FUNC_this_cpu_ptr: { struct bpf_reg_state *reg = ®s[BPF_REG_1]; const struct btf_type *type;
if (reg->type & MEM_RCU) { type = btf_type_by_id(reg->btf, reg->btf_id); if (!type || !btf_type_is_struct(type)) { verbose(env, "Helper has invalid btf/btf_id in R1\n"); return -EFAULT; } returns_cpu_specific_alloc_ptr = true; env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true; } break; } case BPF_FUNC_user_ringbuf_drain: err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_user_ringbuf_callback_state); break; }
if (err) return err;
/* reset caller saved regs */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } invalidate_outgoing_stack_args(env, cur_func(env));
/* helper call returns 64-bit value. */ regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
/* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type);
switch (base_type(ret_type)) { case RET_INTEGER: /* sets type to SCALAR_VALUE */ mark_reg_unknown(env, regs, BPF_REG_0); break; case RET_VOID: regs[BPF_REG_0].type = NOT_INIT; break; case RET_PTR_TO_MAP_VALUE: /* There is no offset yet applied, variable or fixed */ mark_reg_known_zero(env, regs, BPF_REG_0); /* remember map_ptr, so that check_map_access() * can check 'value_size' boundary of memory access * to map element returned from bpf_map_lookup_elem() */ if (meta.map.ptr == NULL) { verifier_bug(env, "unexpected null map_ptr"); return -EFAULT; }
if (func_id == BPF_FUNC_map_lookup_elem && can_elide_value_nullness(meta.map.ptr) && meta.const_map_key >= 0 && meta.const_map_key < meta.map.ptr->max_entries) ret_flag &= ~PTR_MAYBE_NULL;
regs[BPF_REG_0].map_ptr = meta.map.ptr; regs[BPF_REG_0].map_uid = meta.map.uid; regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; if (!type_may_be_null(ret_flag) && btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { regs[BPF_REG_0].id = ++env->id_gen; } break; case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; break; case RET_PTR_TO_SOCK_COMMON: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; break; case RET_PTR_TO_TCP_SOCK: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; break; case RET_PTR_TO_MEM: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; regs[BPF_REG_0].mem_size = meta.mem_size; break; case RET_PTR_TO_MEM_OR_BTF_ID: { const struct btf_type *t;
mark_reg_known_zero(env, regs, BPF_REG_0); t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); if (!btf_type_is_struct(t)) { u32 tsize; const struct btf_type *ret; const char *tname;
/* resolve the type size of ksym. */ ret = btf_resolve_size(meta.ret_btf, t, &tsize); if (IS_ERR(ret)) { tname = btf_name_by_offset(meta.ret_btf, t->name_off); verbose(env, "unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret)); return -EINVAL; } regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; regs[BPF_REG_0].mem_size = tsize; } else { if (returns_cpu_specific_alloc_ptr) { regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU; } else { /* MEM_RDONLY may be carried from ret_flag, but it * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise * it will confuse the check of PTR_TO_BTF_ID in * check_mem_access(). */ ret_flag &= ~MEM_RDONLY; regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; }
regs[BPF_REG_0].btf = meta.ret_btf; regs[BPF_REG_0].btf_id = meta.ret_btf_id; } break; } case RET_PTR_TO_BTF_ID: { struct btf *ret_btf; int ret_btf_id;
mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; if (func_id == BPF_FUNC_kptr_xchg) { ret_btf = meta.kptr_field->kptr.btf; ret_btf_id = meta.kptr_field->kptr.btf_id; if (!btf_is_kernel(ret_btf)) { regs[BPF_REG_0].type |= MEM_ALLOC; if (meta.kptr_field->type == BPF_KPTR_PERCPU) regs[BPF_REG_0].type |= MEM_PERCPU; } } else { if (fn->ret_btf_id == BPF_PTR_POISON) { verifier_bug(env, "func %s has non-overwritten BPF_PTR_POISON return type", func_id_name(func_id)); return -EFAULT; } ret_btf = btf_vmlinux; ret_btf_id = *fn->ret_btf_id; } if (ret_btf_id == 0) { verbose(env, "invalid return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id), func_id); return -EINVAL; } regs[BPF_REG_0].btf = ret_btf; regs[BPF_REG_0].btf_id = ret_btf_id; break; } default: verbose(env, "unknown return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id), func_id); return -EINVAL; }
if (type_may_be_null(regs[BPF_REG_0].type)) regs[BPF_REG_0].id = ++env->id_gen;
if (is_ptr_cast_function(func_id) && find_reference_state(env->cur_state, meta.ref_obj.id)) { struct bpf_verifier_state *branch; struct bpf_reg_state *r0;
err = validate_ref_obj(env, &meta.ref_obj); if (err) return err;
/* * In order for a release of any of the original or cast pointers * to invalidate all other pointers, reuse the same reference id for * the cast result. * This reference id can't be used for nullness propagation, * as cast might return NULL for a non-NULL input. * Hence, explore the NULL case as a separate branch. */ branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch);
r0 = &branch->frame[branch->curframe]->regs[BPF_REG_0]; __mark_reg_known_zero(r0); r0->type = SCALAR_VALUE;
regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; regs[BPF_REG_0].id = meta.ref_obj.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { int id = acquire_reference(env, insn_idx, 0);
if (id < 0) return id;
regs[BPF_REG_0].id = id; }
if (func_id == BPF_FUNC_dynptr_data) regs[BPF_REG_0].parent_id = meta.dynptr.id;
err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); if (err) return err;
err = check_map_func_compatibility(env, meta.map.ptr, func_id); if (err) return err;
if ((func_id == BPF_FUNC_get_stack || func_id == BPF_FUNC_get_task_stack) && !env->prog->has_callchain_buf) { const char *err_str;
#ifdef CONFIG_PERF_EVENTS err = get_callchain_buffers(sysctl_perf_event_max_stack); err_str = "cannot get callchain buffer for func %s#%d\n";#else err = -ENOTSUPP; err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";#endif if (err) { verbose(env, err_str, func_id_name(func_id), func_id); return err; }
env->prog->has_callchain_buf = true; }
if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) env->prog->call_get_stack = true;
if (func_id == BPF_FUNC_get_func_ip) { if (check_get_func_ip(env)) return -ENOTSUPP; env->prog->call_get_func_ip = true; }
if (func_id == BPF_FUNC_tail_call) { if (env->cur_state->curframe) { struct bpf_verifier_state *branch;
mark_reg_scratched(env, BPF_REG_0); branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch); clear_all_pkt_pointers(env); mark_reg_unknown(env, regs, BPF_REG_0); err = prepare_func_exit(env, &env->insn_idx); if (err) return err; env->insn_idx--; } else { changes_data = false; } }
if (changes_data) clear_all_pkt_pointers(env); return 0;}
/* mark_btf_func_reg_size() is used when the reg size is determined by * the BTF func_proto's return value size and argument. */static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, size_t reg_size){ struct bpf_reg_state *reg = ®s[regno];
if (regno == BPF_REG_0) { /* Function return value */ reg->subreg_def = reg_size == sizeof(u64) ? DEF_NOT_SUBREG : env->insn_idx + 1; } else if (reg_size == sizeof(u64)) { /* Function argument */ mark_insn_zext(env, reg); }}
static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, size_t reg_size){ return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size);}
static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_ACQUIRE;}
static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_RELEASE;}
static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_DESTRUCTIVE;}
static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_RCU;}
static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta){ return meta->kfunc_flags & KF_RCU_PROTECTED;}
static bool is_kfunc_arg_mem_size(const struct btf *btf, const struct btf_param *arg, const struct bpf_reg_state *reg){ const struct btf_type *t;
t = btf_type_skip_modifiers(btf, arg->type, NULL); if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) return false;
return btf_param_match_suffix(btf, arg, "__sz");}
static bool is_kfunc_arg_const_mem_size(const struct btf *btf, const struct btf_param *arg, const struct bpf_reg_state *reg){ const struct btf_type *t;
t = btf_type_skip_modifiers(btf, arg->type, NULL); if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) return false;
return btf_param_match_suffix(btf, arg, "__szk");}
static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__k");}
static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__ign");}
static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__map");}
static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__alloc");}
static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__uninit");}
static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__refcounted_kptr");}
static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__nullable");}
static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__nonown_allowed");}
static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__str");}
static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param *arg){ return btf_param_match_suffix(btf, arg, "__irq_flag");}
static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, const struct btf_param *arg, const char *name){ int len, target_len = strlen(name); const char *param_name;
param_name = btf_name_by_offset(btf, arg->name_off); if (str_is_empty(param_name)) return false; len = strlen(param_name); if (len != target_len) return false; if (strcmp(param_name, name)) return false;
return true;}
enum { KF_ARG_DYNPTR_ID, KF_ARG_LIST_HEAD_ID, KF_ARG_LIST_NODE_ID, KF_ARG_RB_ROOT_ID, KF_ARG_RB_NODE_ID, KF_ARG_WORKQUEUE_ID, KF_ARG_RES_SPIN_LOCK_ID, KF_ARG_TASK_WORK_ID, KF_ARG_PROG_AUX_ID, KF_ARG_TIMER_ID};
BTF_ID_LIST(kf_arg_btf_ids)BTF_ID(struct, bpf_dynptr)BTF_ID(struct, bpf_list_head)BTF_ID(struct, bpf_list_node)BTF_ID(struct, bpf_rb_root)BTF_ID(struct, bpf_rb_node)BTF_ID(struct, bpf_wq)BTF_ID(struct, bpf_res_spin_lock)BTF_ID(struct, bpf_task_work)BTF_ID(struct, bpf_prog_aux)BTF_ID(struct, bpf_timer)
static bool __is_kfunc_ptr_arg_type(const struct btf *btf, const struct btf_param *arg, int type){ const struct btf_type *t; u32 res_id;
t = btf_type_skip_modifiers(btf, arg->type, NULL); if (!t) return false; if (!btf_type_is_ptr(t)) return false; t = btf_type_skip_modifiers(btf, t->type, &res_id); if (!t) return false; return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]);}
static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID);}
static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID);}
static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID);}
static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID);}
static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID);}
static bool is_kfunc_arg_timer(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TIMER_ID);}
static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID);}
static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);}
static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);}
static bool is_rbtree_node_type(const struct btf_type *t){ return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_RB_NODE_ID]);}
static bool is_list_node_type(const struct btf_type *t){ return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_LIST_NODE_ID]);}
static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_param *arg){ const struct btf_type *t;
t = btf_type_resolve_func_ptr(btf, arg->type, NULL); if (!t) return false;
return true;}
static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param *arg){ return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_PROG_AUX_ID);}
/* * A kfunc with KF_IMPLICIT_ARGS has two prototypes in BTF: * - the _impl prototype with full arg list (meta->func_proto) * - the BPF API prototype w/o implicit args (func->type in BTF) * To determine whether an argument is implicit, we compare its position * against the number of arguments in the prototype w/o implicit args. */static bool is_kfunc_arg_implicit(const struct bpf_kfunc_call_arg_meta *meta, u32 arg_idx){ const struct btf_type *func, *func_proto; u32 argn;
if (!(meta->kfunc_flags & KF_IMPLICIT_ARGS)) return false;
func = btf_type_by_id(meta->btf, meta->func_id); func_proto = btf_type_by_id(meta->btf, func->type); argn = btf_type_vlen(func_proto);
return argn <= arg_idx;}
/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_type *t, int rec){ const struct btf_type *member_type; const struct btf_member *member; u32 i;
if (!btf_type_is_struct(t)) return false;
for_each_member(i, t, member) { const struct btf_array *array;
member_type = btf_type_skip_modifiers(btf, member->type, NULL); if (btf_type_is_struct(member_type)) { if (rec >= 3) { verbose(env, "max struct nesting depth exceeded\n"); return false; } if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) return false; continue; } if (btf_type_is_array(member_type)) { array = btf_array(member_type); if (!array->nelems) return false; member_type = btf_type_skip_modifiers(btf, array->type, NULL); if (!btf_type_is_scalar(member_type)) return false; continue; } if (!btf_type_is_scalar(member_type)) return false; } return true;}
enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ KF_ARG_PTR_TO_DYNPTR, KF_ARG_PTR_TO_ITER, KF_ARG_PTR_TO_LIST_HEAD, KF_ARG_PTR_TO_LIST_NODE, KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ KF_ARG_PTR_TO_MEM, KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, KF_ARG_PTR_TO_MAP, KF_ARG_PTR_TO_TIMER, KF_ARG_PTR_TO_WORKQUEUE, KF_ARG_PTR_TO_IRQ_FLAG, KF_ARG_PTR_TO_RES_SPIN_LOCK, KF_ARG_PTR_TO_TASK_WORK,};
enum special_kfunc_type { KF_bpf_obj_new_impl, KF_bpf_obj_new, KF_bpf_obj_drop_impl, KF_bpf_obj_drop, KF_bpf_refcount_acquire_impl, KF_bpf_refcount_acquire, KF_bpf_list_push_front_impl, KF_bpf_list_push_front, KF_bpf_list_push_back_impl, KF_bpf_list_push_back, KF_bpf_list_add, KF_bpf_list_pop_front, KF_bpf_list_pop_back, KF_bpf_list_del, KF_bpf_list_front, KF_bpf_list_back, KF_bpf_list_is_first, KF_bpf_list_is_last, KF_bpf_list_empty, KF_bpf_cast_to_kern_ctx, KF_bpf_rdonly_cast, KF_bpf_rcu_read_lock, KF_bpf_rcu_read_unlock, KF_bpf_rbtree_remove, KF_bpf_rbtree_add_impl, KF_bpf_rbtree_add, KF_bpf_rbtree_first, KF_bpf_rbtree_root, KF_bpf_rbtree_left, KF_bpf_rbtree_right, KF_bpf_dynptr_from_skb, KF_bpf_dynptr_from_xdp, KF_bpf_dynptr_from_skb_meta, KF_bpf_xdp_pull_data, KF_bpf_dynptr_slice, KF_bpf_dynptr_slice_rdwr, KF_bpf_dynptr_clone, KF_bpf_percpu_obj_new_impl, KF_bpf_percpu_obj_new, KF_bpf_percpu_obj_drop_impl, KF_bpf_percpu_obj_drop, KF_bpf_throw, KF_bpf_wq_set_callback, KF_bpf_preempt_disable, KF_bpf_preempt_enable, KF_bpf_iter_css_task_new, KF_bpf_session_cookie, KF_bpf_get_kmem_cache, KF_bpf_local_irq_save, KF_bpf_local_irq_restore, KF_bpf_iter_num_new, KF_bpf_iter_num_next, KF_bpf_iter_num_destroy, KF_bpf_set_dentry_xattr, KF_bpf_remove_dentry_xattr, KF_bpf_res_spin_lock, KF_bpf_res_spin_unlock, KF_bpf_res_spin_lock_irqsave, KF_bpf_res_spin_unlock_irqrestore, KF_bpf_dynptr_from_file, KF_bpf_dynptr_file_discard, KF___bpf_trap, KF_bpf_task_work_schedule_signal, KF_bpf_task_work_schedule_resume, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, KF_bpf_arena_reserve_pages, KF_bpf_session_is_return, KF_bpf_stream_vprintk, KF_bpf_stream_print_stack,};
BTF_ID_LIST(special_kfunc_list)BTF_ID(func, bpf_obj_new_impl)BTF_ID(func, bpf_obj_new)BTF_ID(func, bpf_obj_drop_impl)BTF_ID(func, bpf_obj_drop)BTF_ID(func, bpf_refcount_acquire_impl)BTF_ID(func, bpf_refcount_acquire)BTF_ID(func, bpf_list_push_front_impl)BTF_ID(func, bpf_list_push_front)BTF_ID(func, bpf_list_push_back_impl)BTF_ID(func, bpf_list_push_back)BTF_ID(func, bpf_list_add)BTF_ID(func, bpf_list_pop_front)BTF_ID(func, bpf_list_pop_back)BTF_ID(func, bpf_list_del)BTF_ID(func, bpf_list_front)BTF_ID(func, bpf_list_back)BTF_ID(func, bpf_list_is_first)BTF_ID(func, bpf_list_is_last)BTF_ID(func, bpf_list_empty)BTF_ID(func, bpf_cast_to_kern_ctx)BTF_ID(func, bpf_rdonly_cast)BTF_ID(func, bpf_rcu_read_lock)BTF_ID(func, bpf_rcu_read_unlock)BTF_ID(func, bpf_rbtree_remove)BTF_ID(func, bpf_rbtree_add_impl)BTF_ID(func, bpf_rbtree_add)BTF_ID(func, bpf_rbtree_first)BTF_ID(func, bpf_rbtree_root)BTF_ID(func, bpf_rbtree_left)BTF_ID(func, bpf_rbtree_right)#ifdef CONFIG_NETBTF_ID(func, bpf_dynptr_from_skb)BTF_ID(func, bpf_dynptr_from_xdp)BTF_ID(func, bpf_dynptr_from_skb_meta)BTF_ID(func, bpf_xdp_pull_data)#elseBTF_ID_UNUSEDBTF_ID_UNUSEDBTF_ID_UNUSEDBTF_ID_UNUSED#endifBTF_ID(func, bpf_dynptr_slice)BTF_ID(func, bpf_dynptr_slice_rdwr)BTF_ID(func, bpf_dynptr_clone)BTF_ID(func, bpf_percpu_obj_new_impl)BTF_ID(func, bpf_percpu_obj_new)BTF_ID(func, bpf_percpu_obj_drop_impl)BTF_ID(func, bpf_percpu_obj_drop)BTF_ID(func, bpf_throw)BTF_ID(func, bpf_wq_set_callback)BTF_ID(func, bpf_preempt_disable)BTF_ID(func, bpf_preempt_enable)#ifdef CONFIG_CGROUPSBTF_ID(func, bpf_iter_css_task_new)#elseBTF_ID_UNUSED#endif#ifdef CONFIG_BPF_EVENTSBTF_ID(func, bpf_session_cookie)#elseBTF_ID_UNUSED#endifBTF_ID(func, bpf_get_kmem_cache)BTF_ID(func, bpf_local_irq_save)BTF_ID(func, bpf_local_irq_restore)BTF_ID(func, bpf_iter_num_new)BTF_ID(func, bpf_iter_num_next)BTF_ID(func, bpf_iter_num_destroy)#ifdef CONFIG_BPF_LSMBTF_ID(func, bpf_set_dentry_xattr)BTF_ID(func, bpf_remove_dentry_xattr)#elseBTF_ID_UNUSEDBTF_ID_UNUSED#endifBTF_ID(func, bpf_res_spin_lock)BTF_ID(func, bpf_res_spin_unlock)BTF_ID(func, bpf_res_spin_lock_irqsave)BTF_ID(func, bpf_res_spin_unlock_irqrestore)BTF_ID(func, bpf_dynptr_from_file)BTF_ID(func, bpf_dynptr_file_discard)BTF_ID(func, __bpf_trap)BTF_ID(func, bpf_task_work_schedule_signal)BTF_ID(func, bpf_task_work_schedule_resume)BTF_ID(func, bpf_arena_alloc_pages)BTF_ID(func, bpf_arena_free_pages)BTF_ID(func, bpf_arena_reserve_pages)#ifdef CONFIG_BPF_EVENTSBTF_ID(func, bpf_session_is_return)#elseBTF_ID_UNUSED#endifBTF_ID(func, bpf_stream_vprintk)BTF_ID(func, bpf_stream_print_stack)
static bool is_bpf_obj_new_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_obj_new] || func_id == special_kfunc_list[KF_bpf_obj_new_impl];}
static bool is_bpf_percpu_obj_new_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_percpu_obj_new] || func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl];}
static bool is_bpf_obj_drop_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_obj_drop] || func_id == special_kfunc_list[KF_bpf_obj_drop_impl];}
static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_percpu_obj_drop] || func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl];}
static bool is_bpf_refcount_acquire_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_refcount_acquire] || func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];}
static bool is_bpf_list_push_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_list_push_front] || func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || func_id == special_kfunc_list[KF_bpf_list_push_back] || func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || func_id == special_kfunc_list[KF_bpf_list_add];}
static bool is_bpf_rbtree_add_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_rbtree_add] || func_id == special_kfunc_list[KF_bpf_rbtree_add_impl];}
static bool is_task_work_add_kfunc(u32 func_id){ return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume];}
static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta){ if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref) return false;
return meta->kfunc_flags & KF_RET_NULL;}
static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta){ return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];}
static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta){ return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];}
static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta){ return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable];}
static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta){ return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable];}
bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta){ return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data];}
static enum kfunc_ptr_arg_typeget_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta, const struct btf_type *t, const struct btf_type *ref_t, const char *ref_tname, const struct btf_param *args, int arg, int nargs, argno_t argno, struct bpf_reg_state *reg){ bool arg_mem_size = false;
if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) return KF_ARG_PTR_TO_CTX;
if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) arg_mem_size = true;
/* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) return KF_ARG_PTR_TO_CTX;
if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && !arg_mem_size) return KF_ARG_PTR_TO_NULL;
if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) return KF_ARG_PTR_TO_ALLOC_BTF_ID;
if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) return KF_ARG_PTR_TO_REFCOUNTED_KPTR;
if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) return KF_ARG_PTR_TO_DYNPTR;
if (is_kfunc_arg_iter(meta, arg, &args[arg])) return KF_ARG_PTR_TO_ITER;
if (is_kfunc_arg_list_head(meta->btf, &args[arg])) return KF_ARG_PTR_TO_LIST_HEAD;
if (is_kfunc_arg_list_node(meta->btf, &args[arg])) return KF_ARG_PTR_TO_LIST_NODE;
if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RB_ROOT;
if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RB_NODE;
if (is_kfunc_arg_const_str(meta->btf, &args[arg])) return KF_ARG_PTR_TO_CONST_STR;
if (is_kfunc_arg_map(meta->btf, &args[arg])) return KF_ARG_PTR_TO_MAP;
if (is_kfunc_arg_wq(meta->btf, &args[arg])) return KF_ARG_PTR_TO_WORKQUEUE;
if (is_kfunc_arg_timer(meta->btf, &args[arg])) return KF_ARG_PTR_TO_TIMER;
if (is_kfunc_arg_task_work(meta->btf, &args[arg])) return KF_ARG_PTR_TO_TASK_WORK;
if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) return KF_ARG_PTR_TO_IRQ_FLAG;
if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) return KF_ARG_PTR_TO_RES_SPIN_LOCK;
if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { if (!btf_type_is_struct(ref_t)) { verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", meta->func_name, reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } return KF_ARG_PTR_TO_BTF_ID; }
if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) return KF_ARG_PTR_TO_CALLBACK;
/* This is the catch all argument type of register types supported by * check_helper_mem_access. However, we only allow when argument type is * pointer to scalar, or struct composed (recursively) of scalars. When * arg_mem_size is true, the pointer can be void *. */ if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); return -EINVAL; } return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;}
static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const struct btf_type *ref_t, const char *ref_tname, u32 ref_id, struct bpf_kfunc_call_arg_meta *meta, int arg, argno_t argno){ const struct btf_type *reg_ref_t; bool strict_type_match = false; const struct btf *reg_btf; const char *reg_ref_tname; bool taking_projection; bool struct_same; u32 reg_ref_id;
if (base_type(reg->type) == PTR_TO_BTF_ID) { reg_btf = reg->btf; reg_ref_id = reg->btf_id; } else { reg_btf = btf_vmlinux; reg_ref_id = *reg2btf_ids[base_type(reg->type)]; }
/* Enforce strict type matching for calls to kfuncs that are acquiring * or releasing a reference, or are no-cast aliases. We do _not_ * enforce strict matching for kfuncs by default, * as we want to enable BPF programs to pass types that are bitwise * equivalent without forcing them to explicitly cast with something * like bpf_cast_to_kern_ctx(). * * For example, say we had a type like the following: * * struct bpf_cpumask { * cpumask_t cpumask; * refcount_t usage; * }; * * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed * to a struct cpumask, so it would be safe to pass a struct * bpf_cpumask * to a kfunc expecting a struct cpumask *. * * The philosophy here is similar to how we allow scalars of different * types to be passed to kfuncs as long as the size is the same. The * only difference here is that we're simply allowing * btf_struct_ids_match() to walk the struct at the 0th offset, and * resolve types. */ if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) || btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) strict_type_match = true;
WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off));
reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, meta->btf, ref_id, strict_type_match); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. */ taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); if (!taking_projection && !struct_same) { verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno), btf_type_str(reg_ref_t), reg_ref_tname); return -EINVAL; } return 0;}
static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta){ int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save;
if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] || meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) { irq_save = true; if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) kfunc_class = IRQ_LOCK_KFUNC; } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] || meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) { irq_save = false; if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) kfunc_class = IRQ_LOCK_KFUNC; } else { verifier_bug(env, "unknown irq flags kfunc"); return -EFAULT; }
if (irq_save) { if (!is_irq_flag_reg_valid_uninit(env, reg)) { verbose(env, "expected uninitialized irq flag as %s\n", reg_arg_name(env, argno)); return -EINVAL; }
err = check_mem_access(env, env->insn_idx, reg, argno, 0, BPF_DW, BPF_WRITE, -1, false, false); if (err) return err;
err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class); if (err) return err; } else { err = is_irq_flag_reg_valid_init(env, reg); if (err) { verbose(env, "expected an initialized irq flag as %s\n", reg_arg_name(env, argno)); return err; }
spi = irq_flag_get_spi(env, reg); if (spi < 0) return spi;
mark_stack_slots_scratched(env, spi, 1);
err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); if (err) return err; } return 0;}
static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ struct btf_record *rec = reg_btf_record(reg);
if (!env->cur_state->active_locks) { verifier_bug(env, "%s w/o active lock", __func__); return -EFAULT; }
if (type_flag(reg->type) & NON_OWN_REF) { verifier_bug(env, "NON_OWN_REF already set"); return -EFAULT; }
reg->type |= NON_OWN_REF; if (rec->refcount_off >= 0) reg->type |= MEM_RCU;
return 0;}
static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id){ struct bpf_func_state *unused; struct bpf_reg_state *reg;
WARN_ON_ONCE(release_reference_nomark(env->cur_state, id));
bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ if (reg->id == id) { reg->id = 0; ref_set_non_owning(env, reg); } }));
return;}
/* Implementation details: * * Each register points to some region of memory, which we define as an * allocation. Each allocation may embed a bpf_spin_lock which protects any * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same * allocation. The lock and the data it protects are colocated in the same * memory region. * * Hence, everytime a register holds a pointer value pointing to such * allocation, the verifier preserves a unique reg->id for it. * * The verifier remembers the lock 'ptr' and the lock 'id' whenever * bpf_spin_lock is called. * * To enable this, lock state in the verifier captures two values: * active_lock.ptr = Register's type specific pointer * active_lock.id = A unique ID for each register pointer value * * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two * supported register types. * * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of * allocated objects is the reg->btf pointer. * * The active_lock.id is non-unique for maps supporting direct_value_addr, as we * can establish the provenance of the map value statically for each distinct * lookup into such maps. They always contain a single map value hence unique * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. * * So, in case of global variables, they use array maps with max_entries = 1, * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point * into the same map value as max_entries is 1, as described above). * * In case of inner map lookups, the inner map pointer has same map_ptr as the * outer map pointer (in verifier context), but each lookup into an inner map * assigns a fresh reg->id to the lookup, so while lookups into distinct inner * maps from the same outer map share the same map_ptr as active_lock.ptr, they * will get different reg->id assigned to each lookup, hence different * active_lock.id. * * In case of allocated objects, active_lock.ptr is the reg->btf, and the * reg->id is a unique ID preserved after the NULL pointer check on the pointer * returned from bpf_obj_new. Each allocation receives a new reg->id. */static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg){ struct bpf_reference_state *s; void *ptr; u32 id;
switch ((int)reg->type) { case PTR_TO_MAP_VALUE: ptr = reg->map_ptr; break; case PTR_TO_BTF_ID | MEM_ALLOC: ptr = reg->btf; break; default: verifier_bug(env, "unknown reg type for lock check"); return -EFAULT; } id = reg->id;
if (!env->cur_state->active_locks) return -EINVAL; s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr); if (!s) { verbose(env, "held lock and object are not in the same allocation\n"); return -EINVAL; } return 0;}
static bool is_bpf_list_api_kfunc(u32 btf_id){ return is_bpf_list_push_kfunc(btf_id) || btf_id == special_kfunc_list[KF_bpf_list_pop_front] || btf_id == special_kfunc_list[KF_bpf_list_pop_back] || btf_id == special_kfunc_list[KF_bpf_list_del] || btf_id == special_kfunc_list[KF_bpf_list_front] || btf_id == special_kfunc_list[KF_bpf_list_back] || btf_id == special_kfunc_list[KF_bpf_list_is_first] || btf_id == special_kfunc_list[KF_bpf_list_is_last] || btf_id == special_kfunc_list[KF_bpf_list_empty];}
static bool is_bpf_rbtree_api_kfunc(u32 btf_id){ return is_bpf_rbtree_add_kfunc(btf_id) || btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || btf_id == special_kfunc_list[KF_bpf_rbtree_first] || btf_id == special_kfunc_list[KF_bpf_rbtree_root] || btf_id == special_kfunc_list[KF_bpf_rbtree_left] || btf_id == special_kfunc_list[KF_bpf_rbtree_right];}
static bool is_bpf_iter_num_api_kfunc(u32 btf_id){ return btf_id == special_kfunc_list[KF_bpf_iter_num_new] || btf_id == special_kfunc_list[KF_bpf_iter_num_next] || btf_id == special_kfunc_list[KF_bpf_iter_num_destroy];}
static bool is_bpf_graph_api_kfunc(u32 btf_id){ return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) || is_bpf_refcount_acquire_kfunc(btf_id);}
static bool is_bpf_res_spin_lock_kfunc(u32 btf_id){ return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore];}
static bool is_bpf_arena_kfunc(u32 btf_id){ return btf_id == special_kfunc_list[KF_bpf_arena_alloc_pages] || btf_id == special_kfunc_list[KF_bpf_arena_free_pages] || btf_id == special_kfunc_list[KF_bpf_arena_reserve_pages];}
static bool is_bpf_stream_kfunc(u32 btf_id){ return btf_id == special_kfunc_list[KF_bpf_stream_vprintk] || btf_id == special_kfunc_list[KF_bpf_stream_print_stack];}
static bool kfunc_spin_allowed(u32 btf_id){ return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) || is_bpf_res_spin_lock_kfunc(btf_id) || is_bpf_arena_kfunc(btf_id) || is_bpf_stream_kfunc(btf_id);}
static bool is_sync_callback_calling_kfunc(u32 btf_id){ return is_bpf_rbtree_add_kfunc(btf_id);}
static bool is_async_callback_calling_kfunc(u32 btf_id){ return is_bpf_wq_set_callback_kfunc(btf_id) || is_task_work_add_kfunc(btf_id);}
bool bpf_is_throw_kfunc(struct bpf_insn *insn){ return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && insn->imm == special_kfunc_list[KF_bpf_throw];}
static bool is_bpf_wq_set_callback_kfunc(u32 btf_id){ return btf_id == special_kfunc_list[KF_bpf_wq_set_callback];}
static bool is_callback_calling_kfunc(u32 btf_id){ return is_sync_callback_calling_kfunc(btf_id) || is_async_callback_calling_kfunc(btf_id);}
static bool is_rbtree_lock_required_kfunc(u32 btf_id){ return is_bpf_rbtree_api_kfunc(btf_id);}
static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, enum btf_field_type head_field_type, u32 kfunc_btf_id){ bool ret;
switch (head_field_type) { case BPF_LIST_HEAD: ret = is_bpf_list_api_kfunc(kfunc_btf_id); break; case BPF_RB_ROOT: ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id); break; default: verbose(env, "verifier internal error: unexpected graph root argument type %s\n", btf_field_type_name(head_field_type)); return false; }
if (!ret) verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", btf_field_type_name(head_field_type)); return ret;}
static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, enum btf_field_type node_field_type, u32 kfunc_btf_id){ bool ret;
switch (node_field_type) { case BPF_LIST_NODE: ret = is_bpf_list_push_kfunc(kfunc_btf_id) || kfunc_btf_id == special_kfunc_list[KF_bpf_list_del] || kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_first] || kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_last]; break; case BPF_RB_NODE: ret = (is_bpf_rbtree_add_kfunc(kfunc_btf_id) || kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] || kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]); break; default: verbose(env, "verifier internal error: unexpected graph node argument type %s\n", btf_field_type_name(node_field_type)); return false; }
if (!ret) verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", btf_field_type_name(node_field_type)); return ret;}
static int__process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta, enum btf_field_type head_field_type, struct btf_field **head_field){ const char *head_type_name; struct btf_field *field; struct btf_record *rec; u32 head_off;
if (meta->btf != btf_vmlinux) { verifier_bug(env, "unexpected btf mismatch in kfunc call"); return -EFAULT; }
if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) return -EFAULT;
head_type_name = btf_field_type_name(head_field_type); if (!tnum_is_const(reg->var_off)) { verbose(env, "%s doesn't have constant offset. %s has to be at the constant offset\n", reg_arg_name(env, argno), head_type_name); return -EINVAL; }
rec = reg_btf_record(reg); head_off = reg->var_off.value; field = btf_record_find(rec, head_off, head_field_type); if (!field) { verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); return -EINVAL; }
/* All functions require bpf_list_head to be protected using a bpf_spin_lock */ if (check_reg_allocation_locked(env, reg)) { verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", rec->spin_lock_off, head_type_name); return -EINVAL; }
if (*head_field) { verifier_bug(env, "repeating %s arg", head_type_name); return -EFAULT; } *head_field = field; return 0;}
static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta){ return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD, &meta->arg_list_head.field);}
static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta){ return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT, &meta->arg_rbtree_root.field);}
static int__process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta, enum btf_field_type head_field_type, enum btf_field_type node_field_type, struct btf_field **node_field){ const char *node_type_name; const struct btf_type *et, *t; struct btf_field *field; u32 node_off;
if (meta->btf != btf_vmlinux) { verifier_bug(env, "unexpected btf mismatch in kfunc call"); return -EFAULT; }
if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) return -EFAULT;
node_type_name = btf_field_type_name(node_field_type); if (!tnum_is_const(reg->var_off)) { verbose(env, "%s doesn't have constant offset. %s has to be at the constant offset\n", reg_arg_name(env, argno), node_type_name); return -EINVAL; }
node_off = reg->var_off.value; field = reg_find_field_offset(reg, node_off, node_field_type); if (!field) { verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); return -EINVAL; }
field = *node_field;
et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); t = btf_type_by_id(reg->btf, reg->btf_id); if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, field->graph_root.value_btf_id, true)) { verbose(env, "operation on %s expects arg#1 %s at offset=%d " "in struct %s, but arg is at offset=%d in struct %s\n", btf_field_type_name(head_field_type), btf_field_type_name(node_field_type), field->graph_root.node_offset, btf_name_by_offset(field->graph_root.btf, et->name_off), node_off, btf_name_by_offset(reg->btf, t->name_off)); return -EINVAL; } meta->arg_btf = reg->btf; meta->arg_btf_id = reg->btf_id;
if (node_off != field->graph_root.node_offset) { verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", node_off, btf_field_type_name(node_field_type), field->graph_root.node_offset, btf_name_by_offset(field->graph_root.btf, et->name_off)); return -EINVAL; }
return 0;}
static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta){ return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_LIST_HEAD, BPF_LIST_NODE, &meta->arg_list_head.field);}
static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, struct bpf_kfunc_call_arg_meta *meta){ return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_RB_ROOT, BPF_RB_NODE, &meta->arg_rbtree_root.field);}
/* * css_task iter allowlist is needed to avoid dead locking on css_set_lock. * LSM hooks and iters (both sleepable and non-sleepable) are safe. * Any sleepable progs are also safe since bpf_check_attach_target() enforce * them can only be attached to some specific hook points. */static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env){ enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
switch (prog_type) { case BPF_PROG_TYPE_LSM: return true; case BPF_PROG_TYPE_TRACING: if (env->prog->expected_attach_type == BPF_TRACE_ITER) return true; fallthrough; default: return in_sleepable(env); }}
static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, int insn_idx){ const char *func_name = meta->func_name, *ref_tname; struct bpf_func_state *caller = cur_func(env); struct bpf_reg_state *regs = cur_regs(env); const struct btf *btf = meta->btf; const struct btf_param *args; struct btf_record *rec; u32 i, nargs; int ret;
args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); if (nargs > MAX_BPF_FUNC_ARGS) { verbose(env, "Function %s has %d > %d args\n", func_name, nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { verbose(env, "JIT does not support kfunc %s() with %d args\n", func_name, nargs); return -ENOTSUPP; }
ret = check_outgoing_stack_args(env, caller, nargs); if (ret) return ret;
/* Check that BTF function arguments match actual types that the * verifier sees. */ for (i = 0; i < nargs; i++) { struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); const struct btf_type *t, *ref_t, *resolve_ret; enum bpf_arg_type arg_type = ARG_DONTCARE; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type;
if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ if (meta->arg_prog) { verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); return -EFAULT; } if (regno < 0) { verbose(env, "%s prog->aux cannot be a stack argument\n", reg_arg_name(env, argno)); return -EINVAL; } meta->arg_prog = true; cur_aux(env)->arg_prog = regno; continue; }
if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i)) continue;
t = btf_type_skip_modifiers(btf, args[i].type, NULL);
if (btf_type_is_scalar(t)) { if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); return -EINVAL; }
if (is_kfunc_arg_constant(meta->btf, &args[i])) { if (meta->arg_constant.found) { verifier_bug(env, "only one constant argument permitted"); return -EFAULT; } if (!tnum_is_const(reg->var_off)) { verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); return -EINVAL; } if (regno >= 0) ret = mark_chain_precision(env, regno); else ret = mark_stack_arg_precision(env, i); if (ret < 0) return ret; meta->arg_constant.found = true; meta->arg_constant.value = reg->var_off.value; } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { meta->r0_rdonly = true; is_ret_buf_sz = true; } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { is_ret_buf_sz = true; }
if (is_ret_buf_sz) { if (meta->r0_size) { verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); return -EINVAL; }
if (!tnum_is_const(reg->var_off)) { verbose(env, "%s is not a const\n", reg_arg_name(env, argno)); return -EINVAL; }
meta->r0_size = reg->var_off.value; if (regno >= 0) ret = mark_chain_precision(env, regno); else ret = mark_stack_arg_precision(env, i); if (ret) return ret; } continue; }
if (!btf_type_is_ptr(t)) { verbose(env, "Unrecognized %s type %s\n", reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; }
if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && !is_kfunc_arg_nullable(meta->btf, &args[i])) { verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); return -EACCES; }
if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_name, reg_arg_name(env, argno)); return -EINVAL; }
if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg);
ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off);
kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, args, i, nargs, argno, reg); if (kf_arg_type < 0) return kf_arg_type;
switch (kf_arg_type) { case KF_ARG_PTR_TO_NULL: continue; case KF_ARG_PTR_TO_MAP: if (!reg->map_ptr) { verbose(env, "pointer in %s isn't map pointer\n", reg_arg_name(env, argno)); return -EINVAL; } if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || reg->map_ptr->record->task_work_off >= 0)) { /* Use map_uid (which is unique id of inner map) to reject: * inner_map1 = bpf_map_lookup_elem(outer_map, key1) * inner_map2 = bpf_map_lookup_elem(outer_map, key2) * if (inner_map1 && inner_map2) { * wq = bpf_map_lookup_elem(inner_map1); * if (wq) * // mismatch would have been allowed * bpf_wq_init(wq, inner_map2); * } * * Comparing map_ptr is enough to distinguish normal and outer maps. */ if (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid) { if (reg->map_ptr->record->task_work_off >= 0) { verbose(env, "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n", meta->map.uid, reg->map_uid); return -EINVAL; } verbose(env, "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", meta->map.uid, reg->map_uid); return -EINVAL; } } meta->map.ptr = reg->map_ptr; meta->map.uid = reg->map_uid; fallthrough; case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: if (!is_trusted_reg(env, reg)) { if (!is_kfunc_rcu(meta)) { verbose(env, "%s must be referenced or trusted\n", reg_arg_name(env, argno)); return -EINVAL; } if (!is_rcu_reg(reg)) { verbose(env, "%s must be a rcu pointer\n", reg_arg_name(env, argno)); return -EINVAL; } } fallthrough; case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: case KF_ARG_PTR_TO_RB_ROOT: case KF_ARG_PTR_TO_RB_NODE: case KF_ARG_PTR_TO_MEM: case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: case KF_ARG_PTR_TO_TIMER: case KF_ARG_PTR_TO_TASK_WORK: case KF_ARG_PTR_TO_IRQ_FLAG: case KF_ARG_PTR_TO_RES_SPIN_LOCK: break; case KF_ARG_PTR_TO_DYNPTR: arg_type = ARG_PTR_TO_DYNPTR; break; case KF_ARG_PTR_TO_CTX: arg_type = ARG_PTR_TO_CTX; break; case KF_ARG_PTR_TO_REFCOUNTED_KPTR: arg_type = ARG_PTR_TO_BTF_ID; btf_id_fixed_off_ok = false; break; default: verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); return -EFAULT; }
if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; ret = __check_func_arg_reg_off(env, reg, argno, arg_type, btf_id_fixed_off_ok); if (ret < 0) return ret;
switch (kf_arg_type) { case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); return -EINVAL; }
if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); if (ret < 0) return -EINVAL; meta->ret_btf_id = ret; } break; case KF_ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { if (!is_bpf_obj_drop_kfunc(meta->func_id)) { verbose(env, "%s expected for bpf_obj_drop()\n", reg_arg_name(env, argno)); return -EINVAL; } } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { verbose(env, "%s expected for bpf_percpu_obj_drop()\n", reg_arg_name(env, argno)); return -EINVAL; } } else { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } if (meta->btf == btf_vmlinux) { meta->arg_btf = reg->btf; meta->arg_btf_id = reg->btf_id; } break; case KF_ARG_PTR_TO_DYNPTR: { enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
if (is_kfunc_arg_uninit(btf, &args[i])) dynptr_arg_type |= MEM_UNINIT;
if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { dynptr_arg_type |= DYNPTR_TYPE_SKB; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) { dynptr_arg_type |= DYNPTR_TYPE_XDP; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) { dynptr_arg_type |= DYNPTR_TYPE_SKB_META; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { dynptr_arg_type |= DYNPTR_TYPE_FILE; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && (dynptr_arg_type & MEM_UNINIT)) { enum bpf_dynptr_type parent_type = meta->dynptr.type;
if (parent_type == BPF_DYNPTR_TYPE_INVALID) { verifier_bug(env, "no dynptr type for parent of clone"); return -EFAULT; }
dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); }
ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, &meta->ref_obj, &meta->dynptr); if (ret < 0) return ret; break; } case KF_ARG_PTR_TO_ITER: if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) { if (!check_css_task_iter_allowlist(env)) { verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); return -EINVAL; } } ret = process_iter_arg(env, reg, argno, insn_idx, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_LIST_HEAD: if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to map value or allocated object\n", reg_arg_name(env, argno)); return -EINVAL; } if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_RB_ROOT: if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to map value or allocated object\n", reg_arg_name(env, argno)); return -EINVAL; } if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_LIST_NODE: if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { /* Allow bpf_list_front/back return value for * __nonown_allowed list-node arguments. */ goto check_ok; } if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; }check_ok: ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_RB_NODE: if (is_bpf_rbtree_add_kfunc(meta->func_id)) { if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); return -EINVAL; } } else { if (!type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); return -EINVAL; } if (in_rbtree_lock_required_cb(env)) { verbose(env, "%s not allowed in rbtree cb\n", func_name); return -EINVAL; } }
ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_MAP: /* If argument has '__map' suffix expect 'struct bpf_map *' */ ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; ref_t = btf_type_by_id(btf_vmlinux, ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); fallthrough; case KF_ARG_PTR_TO_BTF_ID: /* Only base_type is checked, further checks are done here */ if ((base_type(reg->type) != PTR_TO_BTF_ID || (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && !reg2btf_ids[base_type(reg->type)]) { verbose(env, "%s is %s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); verbose(env, "expected %s or socket\n", reg_type_str(env, base_type(reg->type) | (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); return -EINVAL; } ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_MEM: resolve_ret = btf_resolve_size(btf, ref_t, &type_size); if (IS_ERR(resolve_ret)) { verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } ret = check_mem_reg(env, reg, argno, type_size); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_MEM_SIZE: { struct bpf_reg_state *buff_reg = reg; const struct btf_param *buff_arg = &args[i]; struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); const struct btf_param *size_arg = &args[i + 1]; argno_t next_argno = argno_from_arg(i + 2);
if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, argno, next_argno); if (ret < 0) { verbose(env, "%s and ", reg_arg_name(env, argno)); verbose(env, "%s memory, len pair leads to invalid memory access\n", reg_arg_name(env, next_argno)); return ret; } }
if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { if (meta->arg_constant.found) { verifier_bug(env, "only one constant argument permitted"); return -EFAULT; } if (!tnum_is_const(size_reg->var_off)) { verbose(env, "%s must be a known constant\n", reg_arg_name(env, next_argno)); return -EINVAL; } meta->arg_constant.found = true; meta->arg_constant.value = size_reg->var_off.value; }
/* Skip next '__sz' or '__szk' argument */ i++; break; } case KF_ARG_PTR_TO_CALLBACK: if (reg->type != PTR_TO_FUNC) { verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); return -EINVAL; } meta->subprogno = reg->subprogno; break; case KF_ARG_PTR_TO_REFCOUNTED_KPTR: if (!type_is_ptr_alloc_obj(reg->type)) { verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) meta->arg_owning_ref = true;
rec = reg_btf_record(reg); if (!rec) { verifier_bug(env, "Couldn't find btf_record"); return -EFAULT; }
if (rec->refcount_off < 0) { verbose(env, "%s doesn't point to a type with bpf_refcount field\n", reg_arg_name(env, argno)); return -EINVAL; }
meta->arg_btf = reg->btf; meta->arg_btf_id = reg->btf_id; break; case KF_ARG_PTR_TO_CONST_STR: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a const string\n", reg_arg_name(env, argno)); return -EINVAL; } ret = check_arg_const_str(env, reg, argno); if (ret) return ret; break; case KF_ARG_PTR_TO_WORKQUEUE: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a map value\n", reg_arg_name(env, argno)); return -EINVAL; } ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_TIMER: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a map value\n", reg_arg_name(env, argno)); return -EINVAL; } ret = process_timer_kfunc(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_TASK_WORK: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a map value\n", reg_arg_name(env, argno)); return -EINVAL; } ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_IRQ_FLAG: if (reg->type != PTR_TO_STACK) { verbose(env, "%s doesn't point to an irq flag on stack\n", reg_arg_name(env, argno)); return -EINVAL; } ret = process_irq_flag(env, reg, argno, meta); if (ret < 0) return ret; break; case KF_ARG_PTR_TO_RES_SPIN_LOCK: { int flags = PROCESS_RES_LOCK;
if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "%s doesn't point to map value or allocated object\n", reg_arg_name(env, argno)); return -EINVAL; }
if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) return -EFAULT; if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) flags |= PROCESS_SPIN_LOCK; if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) flags |= PROCESS_LOCK_IRQ; ret = process_spin_lock(env, reg, argno, flags); if (ret < 0) return ret; break; } } }
return 0;}
int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, struct bpf_kfunc_call_arg_meta *meta){ struct bpf_kfunc_meta kfunc; int err;
err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) return err;
memset(meta, 0, sizeof(*meta)); meta->btf = kfunc.btf; meta->func_id = kfunc.id; meta->func_proto = kfunc.proto; meta->func_name = kfunc.name;
if (!kfunc.flags || !btf_kfunc_is_allowed(kfunc.btf, kfunc.id, env->prog)) return -EACCES;
meta->kfunc_flags = *kfunc.flags;
/* Only support release referenced argument passed by register */ if (is_kfunc_release(meta)) meta->release_regno = BPF_REG_1;
return 0;}
/* * Determine how many bytes a helper accesses through a stack pointer at * argument position @arg (0-based, corresponding to R1-R5). * * Returns: * > 0 known read access size in bytes * 0 doesn't read anything directly * S64_MIN unknown * < 0 known write access of (-return) bytes */s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx){ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; const struct bpf_func_proto *fn; enum bpf_arg_type at; s64 size;
if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) return S64_MIN;
at = fn->arg_type[arg];
switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: case ARG_PTR_TO_MAP_VALUE: { bool is_key = base_type(at) == ARG_PTR_TO_MAP_KEY; u64 val; int i, map_reg;
for (i = 0; i < arg; i++) { if (base_type(fn->arg_type[i]) == ARG_CONST_MAP_PTR) break; } if (i >= arg) goto scan_all_maps;
map_reg = BPF_REG_1 + i;
if (!(aux->const_reg_map_mask & BIT(map_reg))) goto scan_all_maps;
i = aux->const_reg_vals[map_reg]; if (i < env->used_map_cnt) { size = is_key ? env->used_maps[i]->key_size : env->used_maps[i]->value_size; goto out; }scan_all_maps: /* * Map pointer is not known at this call site (e.g. different * maps on merged paths). Conservatively return the largest * key_size or value_size across all maps used by the program. */ val = 0; for (i = 0; i < env->used_map_cnt; i++) { struct bpf_map *map = env->used_maps[i]; u32 sz = is_key ? map->key_size : map->value_size;
if (sz > val) val = sz; if (map->inner_map_meta) { sz = is_key ? map->inner_map_meta->key_size : map->inner_map_meta->value_size; if (sz > val) val = sz; } } if (!val) return S64_MIN; size = val; goto out; } case ARG_PTR_TO_MEM: if (at & MEM_FIXED_SIZE) { size = fn->arg_size[arg]; goto out; } if (arg + 1 < ARRAY_SIZE(fn->arg_type) && arg_type_is_mem_size(fn->arg_type[arg + 1])) { int size_reg = BPF_REG_1 + arg + 1;
if (aux->const_reg_mask & BIT(size_reg)) { size = (s64)aux->const_reg_vals[size_reg]; goto out; } /* * Size arg is const on each path but differs across merged * paths. MAX_BPF_STACK is a safe upper bound for reads. */ if (at & MEM_UNINIT) return 0; return MAX_BPF_STACK; } return S64_MIN; case ARG_PTR_TO_DYNPTR: size = BPF_DYNPTR_SIZE; break; case ARG_PTR_TO_STACK: /* * Only used by bpf_calls_callback() helpers. The helper itself * doesn't access stack. The callback subprog does and it's * analyzed separately. */ return 0; default: return S64_MIN; }out: /* * MEM_UNINIT args are write-only: the helper initializes the * buffer without reading it. */ if (at & MEM_UNINIT) return -size; return size;}
/* * Determine how many bytes a kfunc accesses through a stack pointer at * argument position @arg (0-based, corresponding to R1-R5). * * Returns: * > 0 known read access size in bytes * 0 doesn't access memory through that argument (ex: not a pointer) * S64_MIN unknown * < 0 known write access of (-return) bytes */s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, int arg, int insn_idx){ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; struct bpf_kfunc_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; u32 nargs, type_size; s64 size;
if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) return S64_MIN;
btf = meta.btf; args = btf_params(meta.func_proto); nargs = btf_type_vlen(meta.func_proto); if (arg >= nargs) return 0;
t = btf_type_skip_modifiers(btf, args[arg].type, NULL); if (!btf_type_is_ptr(t)) return 0;
/* dynptr: fixed 16-byte on-stack representation */ if (is_kfunc_arg_dynptr(btf, &args[arg])) { size = BPF_DYNPTR_SIZE; goto out; }
/* ptr + __sz/__szk pair: size is in the next register */ if (arg + 1 < nargs && (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1;
if (aux->const_reg_mask & BIT(size_reg)) { size = (s64)aux->const_reg_vals[size_reg]; goto out; } return MAX_BPF_STACK; }
/* fixed-size pointed-to type: resolve via BTF */ ref_t = btf_type_skip_modifiers(btf, t->type, NULL); if (!IS_ERR(btf_resolve_size(btf, ref_t, &type_size))) { size = type_size; goto out; }
return S64_MIN;out: /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) return -size; if (is_kfunc_arg_uninit(btf, &args[arg])) return -size; return size;}
/* check special kfuncs and return: * 1 - not fall-through to 'else' branch, continue verification * 0 - fall-through to 'else' branch * < 0 - not fall-through to 'else' branch, return error */static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux, const struct btf_type *ptr_type, struct btf *desc_btf){ const struct btf_type *ret_t; int err = 0;
if (meta->btf != btf_vmlinux) return 0;
if (is_bpf_obj_new_kfunc(meta->func_id) || is_bpf_percpu_obj_new_kfunc(meta->func_id)) { struct btf_struct_meta *struct_meta; struct btf *ret_btf; u32 ret_btf_id;
if (is_bpf_obj_new_kfunc(meta->func_id) && !bpf_global_ma_set) return -ENOMEM;
if (((u64)(u32)meta->arg_constant.value) != meta->arg_constant.value) { verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); return -EINVAL; }
ret_btf = env->prog->aux->btf; ret_btf_id = meta->arg_constant.value;
/* This may be NULL due to user not supplying a BTF */ if (!ret_btf) { verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n"); return -EINVAL; }
ret_t = btf_type_by_id(ret_btf, ret_btf_id); if (!ret_t || !__btf_type_is_struct(ret_t)) { verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n"); return -EINVAL; }
if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) { verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n", ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE); return -EINVAL; }
if (!bpf_global_percpu_ma_set) { mutex_lock(&bpf_percpu_ma_lock); if (!bpf_global_percpu_ma_set) { /* Charge memory allocated with bpf_global_percpu_ma to * root memcg. The obj_cgroup for root memcg is NULL. */ err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL); if (!err) bpf_global_percpu_ma_set = true; } mutex_unlock(&bpf_percpu_ma_lock); if (err) return err; }
mutex_lock(&bpf_percpu_ma_lock); err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size); mutex_unlock(&bpf_percpu_ma_lock); if (err) return err; }
struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; }
if (struct_meta) { verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n"); return -EINVAL; } }
mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; regs[BPF_REG_0].btf = ret_btf; regs[BPF_REG_0].btf_id = ret_btf_id; if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) regs[BPF_REG_0].type |= MEM_PERCPU;
insn_aux->obj_new_size = ret_t->size; insn_aux->kptr_struct_meta = struct_meta; } else if (is_bpf_refcount_acquire_kfunc(meta->func_id)) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; regs[BPF_REG_0].btf = meta->arg_btf; regs[BPF_REG_0].btf_id = meta->arg_btf_id;
insn_aux->kptr_struct_meta = btf_find_struct_meta(meta->arg_btf, meta->arg_btf_id); } else if (is_list_node_type(ptr_type)) { struct btf_field *field = meta->arg_list_head.field;
mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); } else if (is_rbtree_node_type(ptr_type)) { struct btf_field *field = meta->arg_rbtree_root.field;
mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); } else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].btf_id = meta->ret_btf_id; } else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value); if (!ret_t) { verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n", meta->arg_constant.value); return -EINVAL; } else if (btf_type_is_struct(ret_t)) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].btf_id = meta->arg_constant.value; } else if (btf_type_is_void(ret_t)) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; regs[BPF_REG_0].mem_size = 0; } else { verbose(env, "kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n"); return -EINVAL; } } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] || meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type);
mark_reg_known_zero(env, regs, BPF_REG_0);
if (!meta->arg_constant.found) { verifier_bug(env, "bpf_dynptr_slice(_rdwr) no constant size"); return -EFAULT; }
regs[BPF_REG_0].mem_size = meta->arg_constant.value;
/* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { regs[BPF_REG_0].type |= MEM_RDONLY; } else { /* this will set env->seen_direct_write to true */ if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { verbose(env, "the prog does not allow writes to packet data\n"); return -EINVAL; } }
if (!meta->dynptr.id) { verifier_bug(env, "no dynptr id"); return -EFAULT; } regs[BPF_REG_0].parent_id = meta->dynptr.id; } else { return 0; }
return 1;}
static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p){ bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable; enum bpf_prog_type prog_type = resolve_prog_type(env->prog); struct bpf_reg_state *regs = cur_regs(env); const char *func_name, *ptr_type_name; const struct btf_type *t, *ptr_type; struct bpf_kfunc_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; int err, insn_idx = *insn_idx_p; const struct btf_param *args; u32 i, nargs, ptr_type_id; struct btf *desc_btf; int id;
/* skip for now, but return error when we find this in fixup_kfunc_call */ if (!insn->imm) return 0;
err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); if (err == -EACCES && meta.func_name) verbose(env, "calling kernel function %s is not allowed\n", meta.func_name); if (err) return err; desc_btf = meta.btf; func_name = meta.func_name; insn_aux = &env->insn_aux_data[insn_idx];
insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta);
if (!insn->off && (insn->imm == special_kfunc_list[KF_bpf_res_spin_lock] || insn->imm == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { struct bpf_verifier_state *branch; struct bpf_reg_state *regs;
branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) { verbose(env, "failed to push state for failed lock acquisition\n"); return PTR_ERR(branch); }
regs = branch->frame[branch->curframe]->regs;
/* Clear r0-r5 registers in forked state */ for (i = 0; i < CALLER_SAVED_REGS; i++) bpf_mark_reg_not_init(env, ®s[caller_saved[i]]);
mark_reg_unknown(env, regs, BPF_REG_0); err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1); if (err) { verbose(env, "failed to mark s32 range for retval in forked state for lock\n"); return err; } __mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32)); } else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) { verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n"); return -EFAULT; }
if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); return -EACCES; }
sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); return -EACCES; }
/* Track non-sleepable context for kfuncs, same as for helpers. */ if (!in_sleepable_context(env)) insn_aux->non_sleepable = true;
/* Check the arguments */ err = check_kfunc_args(env, &meta, insn_idx); if (err < 0) return err;
if ((is_bpf_obj_drop_kfunc(meta.func_id) || is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ (prog_type == BPF_PROG_TYPE_TRACING && env->prog->expected_attach_type != BPF_TRACE_ITER && !env->prog->sleepable))) { struct btf_struct_meta *struct_meta;
struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); if (struct_meta && btf_record_has_nmi_unsafe_fields(struct_meta->record)) { verbose(env, "%s cannot be used in tracing programs on types with NMI unsafe fields\n", func_name); return -EINVAL; } }
if (is_bpf_rbtree_add_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_rbtree_add_callback_state); if (err) { verbose(env, "kfunc %s#%d failed callback verification\n", func_name, meta.func_id); return err; } }
if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) { meta.r0_size = sizeof(u64); meta.r0_rdonly = false; }
if (is_bpf_wq_set_callback_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_timer_callback_state); if (err) { verbose(env, "kfunc %s#%d failed callback verification\n", func_name, meta.func_id); return err; } }
if (is_task_work_add_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_task_work_schedule_callback_state); if (err) { verbose(env, "kfunc %s#%d failed callback verification\n", func_name, meta.func_id); return err; } }
rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
preempt_disable = is_kfunc_bpf_preempt_disable(&meta); preempt_enable = is_kfunc_bpf_preempt_enable(&meta);
if (rcu_lock) { env->cur_state->active_rcu_locks++; } else if (rcu_unlock) { if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); return -EINVAL; } if (--env->cur_state->active_rcu_locks == 0) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; } else if (preempt_enable) { if (env->cur_state->active_preempt_locks == 0) { verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); return -EINVAL; } env->cur_state->active_preempt_locks--; }
if (sleepable && !in_sleepable_context(env)) { verbose(env, "kernel func %s is sleepable within %s\n", func_name, non_sleepable_context_description(env)); return -EACCES; }
if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); return -EACCES; }
if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) { verbose(env, "kernel func %s requires RCU critical section protection\n", func_name); return -EACCES; }
/* In case of release function, we get register number of refcounted * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. */ if (meta.release_regno) { err = release_reg(env, ®s[meta.release_regno], false, !!meta.dynptr.id); if (err) return err; }
if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) { id = regs[BPF_REG_2].id; insn_aux->insert_off = regs[BPF_REG_2].var_off.value; insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); ref_convert_owning_non_owning(env, id); }
if (meta.func_id == special_kfunc_list[KF_bpf_throw]) { if (!bpf_jit_supports_exceptions()) { verbose(env, "JIT does not support calling kfunc %s#%d\n", func_name, meta.func_id); return -ENOTSUPP; } env->seen_exception = true;
/* In the case of the default callback, the cookie value passed * to bpf_throw becomes the return value of the program. */ if (!env->exception_callback_subprog) { err = check_return_code(env, BPF_REG_1, "R1"); if (err < 0) return err; } }
for (i = 0; i < CALLER_SAVED_REGS; i++) { u32 regno = caller_saved[i];
bpf_mark_reg_not_init(env, ®s[regno]); regs[regno].subreg_def = DEF_NOT_SUBREG; } invalidate_outgoing_stack_args(env, cur_func(env));
/* Check return type */ t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);
if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { if (meta.btf != btf_vmlinux || (!is_bpf_obj_new_kfunc(meta.func_id) && !is_bpf_percpu_obj_new_kfunc(meta.func_id) && !is_bpf_refcount_acquire_kfunc(meta.func_id))) { verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); return -EINVAL; } }
if (btf_type_is_scalar(t)) { mark_reg_unknown(env, regs, BPF_REG_0); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); mark_btf_func_reg_size(env, BPF_REG_0, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); if (err) { if (err < 0) return err; } else if (btf_type_is_void(ptr_type)) { /* kfunc returning 'void *' is equivalent to returning scalar */ mark_reg_unknown(env, regs, BPF_REG_0); } else if (!__btf_type_is_struct(ptr_type)) { if (!meta.r0_size) { __u32 sz;
if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { meta.r0_size = sz; meta.r0_rdonly = true; } } if (!meta.r0_size) { ptr_type_name = btf_name_by_offset(desc_btf, ptr_type->name_off); verbose(env, "kernel function %s returns pointer type %s %s is not supported\n", func_name, btf_type_str(ptr_type), ptr_type_name); return -EINVAL; }
mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM; regs[BPF_REG_0].mem_size = meta.r0_size;
if (meta.r0_rdonly) regs[BPF_REG_0].type |= MEM_RDONLY;
/* Ensures we don't access the memory after a release_reference() */ if (meta.ref_obj.id) { err = validate_ref_obj(env, &meta.ref_obj); if (err) return err; regs[BPF_REG_0].parent_id = meta.ref_obj.id; }
if (is_kfunc_rcu_protected(&meta)) regs[BPF_REG_0].type |= MEM_RCU; } else { enum bpf_reg_type type = PTR_TO_BTF_ID;
if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache]) type |= PTR_UNTRUSTED; else if (is_kfunc_rcu_protected(&meta) || (bpf_is_iter_next_kfunc(&meta) && (get_iter_from_state(env->cur_state, &meta) ->type & MEM_RCU))) { /* * If the iterator's constructor (the _new * function e.g., bpf_iter_task_new) has been * annotated with BPF kfunc flag * KF_RCU_PROTECTED and was called within a RCU * read-side critical section, also propagate * the MEM_RCU flag to the pointer returned from * the iterator's next function (e.g., * bpf_iter_task_next). */ type |= MEM_RCU; } else { /* * Any PTR_TO_BTF_ID that is returned from a BPF * kfunc should by default be treated as * implicitly trusted. */ type |= PTR_TRUSTED; }
mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = type; regs[BPF_REG_0].btf_id = ptr_type_id; }
if (is_kfunc_ret_null(&meta)) { regs[BPF_REG_0].type |= PTR_MAYBE_NULL; /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ regs[BPF_REG_0].id = ++env->id_gen; } mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); if (is_kfunc_acquire(&meta)) { id = acquire_reference(env, insn_idx, 0); if (id < 0) return id; regs[BPF_REG_0].id = id; } else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) { ref_set_non_owning(env, ®s[BPF_REG_0]); }
if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) regs[BPF_REG_0].id = ++env->id_gen; } else if (btf_type_is_void(t)) { if (meta.btf == btf_vmlinux) { if (is_bpf_obj_drop_kfunc(meta.func_id) || is_bpf_percpu_obj_drop_kfunc(meta.func_id)) { insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); } } }
if (bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env);
nargs = btf_type_vlen(meta.func_proto); if (nargs > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt;
if (stack_arg_cnt > caller_info->stack_arg_cnt) caller_info->stack_arg_cnt = stack_arg_cnt; }
args = (const struct btf_param *)(meta.func_proto + 1); for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) { u32 regno = i + 1;
t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); if (btf_type_is_ptr(t)) mark_btf_func_reg_size(env, regno, sizeof(void *)); else /* scalar. ensured by check_kfunc_args() */ mark_btf_func_reg_size(env, regno, t->size); }
if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); if (err) return err; }
if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) env->prog->call_session_cookie = true;
if (bpf_is_throw_kfunc(insn)) return process_bpf_exit_full(env, NULL, true);
return 0;}
static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, enum bpf_reg_type type){ bool known = tnum_is_const(reg->var_off); s64 val = reg->var_off.value; s64 smin = reg_smin(reg);
if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { verbose(env, "math between %s pointer and %lld is not allowed\n", reg_type_str(env, type), val); return false; }
if (smin == S64_MIN) { verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", reg_type_str(env, type)); return false; }
if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { verbose(env, "value %lld makes %s pointer be out of bounds\n", smin, reg_type_str(env, type)); return false; }
return true;}
static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, enum bpf_reg_type type){ bool known = tnum_is_const(reg->var_off); s64 val = reg->var_off.value; s64 smin = reg_smin(reg);
if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { verbose(env, "%s pointer offset %lld is not allowed\n", reg_type_str(env, type), val); return false; }
if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { verbose(env, "%s pointer offset %lld is not allowed\n", reg_type_str(env, type), smin); return false; }
return true;}
enum { REASON_BOUNDS = -1, REASON_TYPE = -2, REASON_PATHS = -3, REASON_LIMIT = -4, REASON_STACK = -5,};
static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, u32 *alu_limit, bool mask_to_left){ u32 max = 0, ptr_limit = 0;
switch (ptr_reg->type) { case PTR_TO_STACK: /* Offset 0 is out-of-bounds, but acceptable start for the * left direction, see BPF_REG_FP. Also, unknown scalar * offset where we would need to deal with min/max bounds is * currently prohibited for unprivileged. */ max = MAX_BPF_STACK + mask_to_left; ptr_limit = -ptr_reg->var_off.value; break; case PTR_TO_MAP_VALUE: max = ptr_reg->map_ptr->value_size; ptr_limit = mask_to_left ? reg_smin(ptr_reg) : reg_umax(ptr_reg); break; default: return REASON_TYPE; }
if (ptr_limit >= max) return REASON_LIMIT; *alu_limit = ptr_limit; return 0;}
static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, const struct bpf_insn *insn){ return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K || cur_aux(env)->nospec;}
static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, u32 alu_state, u32 alu_limit){ /* If we arrived here from different branches with different * state or limits to sanitize, then this won't work. */ if (aux->alu_state && (aux->alu_state != alu_state || aux->alu_limit != alu_limit)) return REASON_PATHS;
/* Corresponding fixup done in do_misc_fixups(). */ aux->alu_state = alu_state; aux->alu_limit = alu_limit; return 0;}
static int sanitize_val_alu(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_insn_aux_data *aux = cur_aux(env);
if (can_skip_alu_sanitation(env, insn)) return 0;
return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);}
static bool sanitize_needed(u8 opcode){ return opcode == BPF_ADD || opcode == BPF_SUB;}
struct bpf_sanitize_info { struct bpf_insn_aux_data aux; bool mask_to_left;};
static int sanitize_speculative_path(struct bpf_verifier_env *env, const struct bpf_insn *insn, u32 next_idx, u32 curr_idx){ struct bpf_verifier_state *branch; struct bpf_reg_state *regs;
branch = push_stack(env, next_idx, curr_idx, true); if (!IS_ERR(branch) && insn) { regs = branch->frame[branch->curframe]->regs; if (BPF_SRC(insn->code) == BPF_K) { mark_reg_unknown(env, regs, insn->dst_reg); } else if (BPF_SRC(insn->code) == BPF_X) { mark_reg_unknown(env, regs, insn->dst_reg); mark_reg_unknown(env, regs, insn->src_reg); } } return PTR_ERR_OR_ZERO(branch);}
static int sanitize_ptr_alu(struct bpf_verifier_env *env, struct bpf_insn *insn, const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg, struct bpf_reg_state *dst_reg, struct bpf_sanitize_info *info, const bool commit_window){ struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; struct bpf_verifier_state *vstate = env->cur_state; bool off_is_imm = tnum_is_const(off_reg->var_off); bool off_is_neg = reg_smin(off_reg) < 0; bool ptr_is_dst_reg = ptr_reg == dst_reg; u8 opcode = BPF_OP(insn->code); u32 alu_state, alu_limit; struct bpf_reg_state tmp; int err;
if (can_skip_alu_sanitation(env, insn)) return 0;
/* We already marked aux for masking from non-speculative * paths, thus we got here in the first place. We only care * to explore bad access from here. */ if (vstate->speculative) goto do_sim;
if (!commit_window) { if (!tnum_is_const(off_reg->var_off) && (reg_smin(off_reg) < 0) != (reg_smax(off_reg) < 0)) return REASON_BOUNDS;
info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || (opcode == BPF_SUB && !off_is_neg); }
err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); if (err < 0) return err;
if (commit_window) { /* In commit phase we narrow the masking window based on * the observed pointer move after the simulated operation. */ alu_state = info->aux.alu_state; alu_limit = abs(info->aux.alu_limit - alu_limit); } else { alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; alu_state |= ptr_is_dst_reg ? BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
/* Limit pruning on unknown scalars to enable deep search for * potential masking differences from other program paths. */ if (!off_is_imm) env->explore_alu_limits = true; }
err = update_alu_sanitation_state(aux, alu_state, alu_limit); if (err < 0) return err;do_sim: /* If we're in commit phase, we're done here given we already * pushed the truncated dst_reg into the speculative verification * stack. * * Also, when register is a known constant, we rewrite register-based * operation to immediate-based, and thus do not need masking (and as * a consequence, do not need to simulate the zero-truncation either). */ if (commit_window || off_is_imm) return 0;
/* Simulate and find potential out-of-bounds access under * speculative execution from truncation as a result of * masking when off was not within expected range. If off * sits in dst, then we temporarily need to move ptr there * to simulate dst (== 0) +/-= ptr. Needed, for example, * for cases where we use K-based arithmetic in one direction * and truncated reg-based in the other in order to explore * bad access. */ if (!ptr_is_dst_reg) { tmp = *dst_reg; *dst_reg = *ptr_reg; } err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx); if (err < 0) return REASON_STACK; if (!ptr_is_dst_reg) *dst_reg = tmp; return 0;}
static void sanitize_mark_insn_seen(struct bpf_verifier_env *env){ struct bpf_verifier_state *vstate = env->cur_state;
/* If we simulate paths under speculation, we don't update the * insn as 'seen' such that when we verify unreachable paths in * the non-speculative domain, sanitize_dead_code() can still * rewrite/sanitize them. */ if (!vstate->speculative) env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;}
static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason, const struct bpf_reg_state *off_reg, const struct bpf_reg_state *dst_reg){ static const char *err = "pointer arithmetic with it prohibited for !root"; const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; u32 dst = insn->dst_reg, src = insn->src_reg;
switch (reason) { case REASON_BOUNDS: verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", off_reg == dst_reg ? dst : src, err); break; case REASON_TYPE: verbose(env, "R%d has pointer with unsupported alu operation, %s\n", off_reg == dst_reg ? src : dst, err); break; case REASON_PATHS: verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", dst, op, err); break; case REASON_LIMIT: verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", dst, op, err); break; case REASON_STACK: verbose(env, "R%d could not be pushed for speculative verification, %s\n", dst, err); return -ENOMEM; default: verifier_bug(env, "unknown reason (%d)", reason); break; }
return -EACCES;}
/* check that stack access falls within stack limits and that 'reg' doesn't * have a variable offset. * * Variable offset is prohibited for unprivileged mode for simplicity since it * requires corresponding support in Spectre masking for stack ALU. See also * retrieve_ptr_limit(). */static int check_stack_access_for_ptr_arithmetic( struct bpf_verifier_env *env, int regno, const struct bpf_reg_state *reg, int off){ if (!tnum_is_const(reg->var_off)) { char tn_buf[48];
tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", regno, tn_buf, off); return -EACCES; }
if (off >= 0 || off < -MAX_BPF_STACK) { verbose(env, "R%d stack pointer arithmetic goes out of range, " "prohibited for !root; off=%d\n", regno, off); return -EACCES; }
return 0;}
static int sanitize_check_bounds(struct bpf_verifier_env *env, const struct bpf_insn *insn, struct bpf_reg_state *dst_reg){ u32 dst = insn->dst_reg;
/* For unprivileged we require that resulting offset must be in bounds * in order to be able to sanitize access later on. */ if (env->bypass_spec_v1) return 0;
switch (dst_reg->type) { case PTR_TO_STACK: if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, dst_reg->var_off.value)) return -EACCES; break; case PTR_TO_MAP_VALUE: if (check_map_access(env, dst_reg, argno_from_reg(dst), 0, 1, false, ACCESS_HELPER)) { verbose(env, "R%d pointer arithmetic of map value goes out of range, " "prohibited for !root\n", dst); return -EACCES; } break; default: return -EOPNOTSUPP; }
return 0;}
/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. * Caller should also handle BPF_MOV case separately. * If we return -EACCES, caller may want to try again treating pointer as a * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. */static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *dst_reg; bool known = tnum_is_const(off_reg->var_off); s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg); u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg); struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; int ret, bounds_ret;
dst_reg = ®s[dst];
if ((known && (smin_val != smax_val || umin_val != umax_val)) || smin_val > smax_val || umin_val > umax_val) { /* Taint dst register if offset had invalid bounds derived from * e.g. dead branches. */ __mark_reg_unknown(env, dst_reg); return 0; }
if (BPF_CLASS(insn->code) != BPF_ALU64) { /* 32-bit ALU ops on pointers produce (meaningless) scalars */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { __mark_reg_unknown(env, dst_reg); return 0; }
verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst); return -EACCES; }
if (ptr_reg->type & PTR_MAYBE_NULL) { verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst, reg_type_str(env, ptr_reg->type)); return -EACCES; }
/* * Accesses to untrusted PTR_TO_MEM are done through probe * instructions, hence no need to track offsets. */ if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) return 0;
switch (base_type(ptr_reg->type)) { case PTR_TO_CTX: case PTR_TO_MAP_VALUE: case PTR_TO_MAP_KEY: case PTR_TO_STACK: case PTR_TO_PACKET_META: case PTR_TO_PACKET: case PTR_TO_TP_BUFFER: case PTR_TO_BTF_ID: case PTR_TO_MEM: case PTR_TO_BUF: case PTR_TO_FUNC: case CONST_PTR_TO_DYNPTR: break; case PTR_TO_FLOW_KEYS: if (known) break; fallthrough; case CONST_PTR_TO_MAP: /* smin_val represents the known value */ if (known && smin_val == 0 && opcode == BPF_ADD) break; fallthrough; default: verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type)); return -EACCES; }
/* In case of 'scalar += pointer', dst_reg inherits pointer type and id. * The id may be overwritten later if we create a new variable offset. */ dst_reg->type = ptr_reg->type; dst_reg->id = ptr_reg->id;
if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) return -EINVAL;
/* pointer types do not carry 32-bit bounds at the moment. */ __mark_reg32_unbounded(dst_reg);
if (sanitize_needed(opcode)) { ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); if (ret < 0) return sanitize_err(env, insn, ret, off_reg, dst_reg); }
switch (opcode) { case BPF_ADD: /* * dst_reg gets the pointer type and since some positive * integer value was added to the pointer, give it a new 'id' * if it's a PTR_TO_PACKET. * this creates a new 'base' pointer, off_reg (variable) gets * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; if (reg_is_pkt_pointer(ptr_reg)) { if (!known) dst_reg->id = ++env->id_gen; /* * Clear range for unknown addends since we can't know * where the pkt pointer ended up. Also clear AT_PKT_END / * BEYOND_PKT_END from prior comparison as any pointer * arithmetic invalidates them. */ if (!known || dst_reg->range < 0) memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); } break; case BPF_SUB: if (dst_reg == off_reg) { /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); return -EACCES; } /* We don't allow subtraction from FP, because (according to * test_verifier.c test "invalid fp arithmetic", JITs might not * be able to deal with it. */ if (ptr_reg->type == PTR_TO_STACK) { verbose(env, "R%d subtraction from stack pointer prohibited\n", dst); return -EACCES; } dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; if (reg_is_pkt_pointer(ptr_reg)) { if (!known) dst_reg->id = ++env->id_gen; /* * Clear range if the subtrahend may be negative since * pkt pointer could move past its bounds. A positive * subtrahend moves it backwards keeping positive range * intact. Also clear AT_PKT_END / BEYOND_PKT_END from * prior comparison as arithmetic invalidates them. */ if ((!known && smin_val < 0) || dst_reg->range < 0) memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); } break; case BPF_AND: case BPF_OR: case BPF_XOR: /* bitwise ops on pointers are troublesome, prohibit. */ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 4]); return -EACCES; default: /* other operators (e.g. MUL,LSH) produce non-pointer results */ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >> 4]); return -EACCES; }
if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) return -EINVAL; reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) return bounds_ret; if (sanitize_needed(opcode)) { ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, &info, true); if (verifier_bug_if(!can_skip_alu_sanitation(env, insn) && !env->cur_state->speculative && bounds_ret && !ret, env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) { return -EFAULT; } if (ret < 0) return sanitize_err(env, insn, ret, off_reg, dst_reg); }
return 0;}
static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ dst_reg->r32 = cnum32_add(dst_reg->r32, src_reg->r32);}
static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ dst_reg->r64 = cnum64_add(dst_reg->r64, src_reg->r64);}
static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ dst_reg->r32 = cnum32_add(dst_reg->r32, cnum32_negate(src_reg->r32));}
static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ dst_reg->r64 = cnum64_add(dst_reg->r64, cnum64_negate(src_reg->r64));}
static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ s32 smin = reg_s32_min(dst_reg); s32 smax = reg_s32_max(dst_reg); u32 umin = reg_u32_min(dst_reg); u32 umax = reg_u32_max(dst_reg); s32 tmp_prod[4];
if (check_mul_overflow(umax, reg_u32_max(src_reg), &umax) || check_mul_overflow(umin, reg_u32_min(src_reg), &umin)) { /* Overflow possible, we know nothing */ umin = 0; umax = U32_MAX; } if (check_mul_overflow(smin, reg_s32_min(src_reg), &tmp_prod[0]) || check_mul_overflow(smin, reg_s32_max(src_reg), &tmp_prod[1]) || check_mul_overflow(smax, reg_s32_min(src_reg), &tmp_prod[2]) || check_mul_overflow(smax, reg_s32_max(src_reg), &tmp_prod[3])) { /* Overflow possible, we know nothing */ smin = S32_MIN; smax = S32_MAX; } else { smin = min_array(tmp_prod, 4); smax = max_array(tmp_prod, 4); }
dst_reg->r32 = cnum32_intersect(cnum32_from_urange(umin, umax), cnum32_from_srange(smin, smax));}
static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ s64 smin = reg_smin(dst_reg); s64 smax = reg_smax(dst_reg); u64 umin = reg_umin(dst_reg); u64 umax = reg_umax(dst_reg); s64 tmp_prod[4];
if (check_mul_overflow(umax, reg_umax(src_reg), &umax) || check_mul_overflow(umin, reg_umin(src_reg), &umin)) { /* Overflow possible, we know nothing */ umin = 0; umax = U64_MAX; } if (check_mul_overflow(smin, reg_smin(src_reg), &tmp_prod[0]) || check_mul_overflow(smin, reg_smax(src_reg), &tmp_prod[1]) || check_mul_overflow(smax, reg_smin(src_reg), &tmp_prod[2]) || check_mul_overflow(smax, reg_smax(src_reg), &tmp_prod[3])) { /* Overflow possible, we know nothing */ smin = S64_MIN; smax = S64_MAX; } else { smin = min_array(tmp_prod, 4); smax = max_array(tmp_prod, 4); }
dst_reg->r64 = cnum64_intersect(cnum64_from_urange(umin, umax), cnum64_from_srange(smin, smax));}
static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */
reg_set_urange32(dst_reg, reg_u32_min(dst_reg) / src_val, reg_u32_max(dst_reg) / src_val);
/* Reset other ranges/tnum to unbounded/unknown. */ reset_reg64_and_tnum(dst_reg);}
static void scalar_min_max_udiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */
reg_set_urange64(dst_reg, div64_u64(reg_umin(dst_reg), src_val), div64_u64(reg_umax(dst_reg), src_val));
/* Reset other ranges/tnum to unbounded/unknown. */ reset_reg32_and_tnum(dst_reg);}
static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ s32 smin = reg_s32_min(dst_reg); s32 smax = reg_s32_max(dst_reg); s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */ s32 res1, res2;
/* BPF div specification: S32_MIN / -1 = S32_MIN */ if (smin == S32_MIN && src_val == -1) { /* * If the dividend range contains more than just S32_MIN, * we cannot precisely track the result, so it becomes unbounded. * e.g., [S32_MIN, S32_MIN+10]/(-1), * = {S32_MIN} U [-(S32_MIN+10), -(S32_MIN+1)] * = {S32_MIN} U [S32_MAX-9, S32_MAX] = [S32_MIN, S32_MAX] * Otherwise (if dividend is exactly S32_MIN), result remains S32_MIN. */ if (smax != S32_MIN) { smin = S32_MIN; smax = S32_MAX; } goto reset; }
res1 = smin / src_val; res2 = smax / src_val; smin = min(res1, res2); smax = max(res1, res2);
reset: reg_set_srange32(dst_reg, smin, smax); /* Reset other ranges/tnum to unbounded/unknown. */ reset_reg64_and_tnum(dst_reg);}
static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ s64 smin = reg_smin(dst_reg); s64 smax = reg_smax(dst_reg); s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */ s64 res1, res2;
/* BPF div specification: S64_MIN / -1 = S64_MIN */ if (smin == S64_MIN && src_val == -1) { /* * If the dividend range contains more than just S64_MIN, * we cannot precisely track the result, so it becomes unbounded. * e.g., [S64_MIN, S64_MIN+10]/(-1), * = {S64_MIN} U [-(S64_MIN+10), -(S64_MIN+1)] * = {S64_MIN} U [S64_MAX-9, S64_MAX] = [S64_MIN, S64_MAX] * Otherwise (if dividend is exactly S64_MIN), result remains S64_MIN. */ if (smax != S64_MIN) { smin = S64_MIN; smax = S64_MAX; } goto reset; }
res1 = div64_s64(smin, src_val); res2 = div64_s64(smax, src_val); smin = min(res1, res2); smax = max(res1, res2);
reset: reg_set_srange64(dst_reg, smin, smax); /* Reset other ranges/tnum to unbounded/unknown. */ reset_reg32_and_tnum(dst_reg);}
static void scalar32_min_max_umod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */ u32 res_max = src_val - 1;
/* * If dst_umax <= res_max, the result remains unchanged. * e.g., [2, 5] % 10 = [2, 5]. */ if (reg_u32_max(dst_reg) <= res_max) return;
reg_set_urange32(dst_reg, 0, min(reg_u32_max(dst_reg), res_max));
/* Reset other ranges/tnum to unbounded/unknown. */ reset_reg64_and_tnum(dst_reg);}
static void scalar_min_max_umod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */ u64 res_max = src_val - 1;
/* * If dst_umax <= res_max, the result remains unchanged. * e.g., [2, 5] % 10 = [2, 5]. */ if (reg_umax(dst_reg) <= res_max) return;
reg_set_urange64(dst_reg, 0, min(reg_umax(dst_reg), res_max));
/* Reset other ranges/tnum to unbounded/unknown. */ reset_reg32_and_tnum(dst_reg);}
static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */
/* * Safe absolute value calculation: * If src_val == S32_MIN (-2147483648), src_abs becomes 2147483648. * Here use unsigned integer to avoid overflow. */ u32 src_abs = (src_val > 0) ? (u32)src_val : -(u32)src_val;
/* * Calculate the maximum possible absolute value of the result. * Even if src_abs is 2147483648 (S32_MIN), subtracting 1 gives * 2147483647 (S32_MAX), which fits perfectly in s32. */ s32 res_max_abs = src_abs - 1;
/* * If the dividend is already within the result range, * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5]. */ if (reg_s32_min(dst_reg) >= -res_max_abs && reg_s32_max(dst_reg) <= res_max_abs) return;
/* General case: result has the same sign as the dividend. */ if (reg_s32_min(dst_reg) >= 0) { reg_set_srange32(dst_reg, 0, min(reg_s32_max(dst_reg), res_max_abs)); } else if (reg_s32_max(dst_reg) <= 0) { reg_set_srange32(dst_reg, max(reg_s32_min(dst_reg), -res_max_abs), 0); } else { reg_set_srange32(dst_reg, -res_max_abs, res_max_abs); }
/* Reset other ranges/tnum to unbounded/unknown. */ reset_reg64_and_tnum(dst_reg);}
static void scalar_min_max_smod(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */
/* * Safe absolute value calculation: * If src_val == S64_MIN (-2^63), src_abs becomes 2^63. * Here use unsigned integer to avoid overflow. */ u64 src_abs = (src_val > 0) ? (u64)src_val : -(u64)src_val;
/* * Calculate the maximum possible absolute value of the result. * Even if src_abs is 2^63 (S64_MIN), subtracting 1 gives * 2^63 - 1 (S64_MAX), which fits perfectly in s64. */ s64 res_max_abs = src_abs - 1;
/* * If the dividend is already within the result range, * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5]. */ if (reg_smin(dst_reg) >= -res_max_abs && reg_smax(dst_reg) <= res_max_abs) return;
/* General case: result has the same sign as the dividend. */ if (reg_smin(dst_reg) >= 0) { reg_set_srange64(dst_reg, 0, min(reg_smax(dst_reg), res_max_abs)); } else if (reg_smax(dst_reg) <= 0) { reg_set_srange64(dst_reg, max(reg_smin(dst_reg), -res_max_abs), 0); } else { reg_set_srange64(dst_reg, -res_max_abs, res_max_abs); }
/* Reset other ranges/tnum to unbounded/unknown. */ reset_reg32_and_tnum(dst_reg);}
static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ bool src_known = tnum_subreg_is_const(src_reg->var_off); bool dst_known = tnum_subreg_is_const(dst_reg->var_off); struct tnum var32_off = tnum_subreg(dst_reg->var_off); u32 umax_val = reg_u32_max(src_reg);
if (src_known && dst_known) { __mark_reg32_known(dst_reg, var32_off.value); return; }
/* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. */ reg_set_urange32(dst_reg, var32_off.value, min(reg_u32_max(dst_reg), umax_val));}
static void scalar_min_max_and(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ bool src_known = tnum_is_const(src_reg->var_off); bool dst_known = tnum_is_const(dst_reg->var_off); u64 umax_val = reg_umax(src_reg);
if (src_known && dst_known) { __mark_reg_known(dst_reg, dst_reg->var_off.value); return; }
/* We get our minimum from the var_off, since that's inherently * bitwise. Our maximum is the minimum of the operands' maxima. */ reg_set_urange64(dst_reg, dst_reg->var_off.value, min(reg_umax(dst_reg), umax_val));
/* We may learn something more from the var_off */ __update_reg_bounds(dst_reg);}
static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ bool src_known = tnum_subreg_is_const(src_reg->var_off); bool dst_known = tnum_subreg_is_const(dst_reg->var_off); struct tnum var32_off = tnum_subreg(dst_reg->var_off); u32 umin_val = reg_u32_min(src_reg);
if (src_known && dst_known) { __mark_reg32_known(dst_reg, var32_off.value); return; }
/* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima */ reg_set_urange32(dst_reg, max(reg_u32_min(dst_reg), umin_val), var32_off.value | var32_off.mask);}
static void scalar_min_max_or(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ bool src_known = tnum_is_const(src_reg->var_off); bool dst_known = tnum_is_const(dst_reg->var_off); u64 umin_val = reg_umin(src_reg);
if (src_known && dst_known) { __mark_reg_known(dst_reg, dst_reg->var_off.value); return; }
/* We get our maximum from the var_off, and our minimum is the * maximum of the operands' minima */ reg_set_urange64(dst_reg, max(reg_umin(dst_reg), umin_val), dst_reg->var_off.value | dst_reg->var_off.mask);
/* We may learn something more from the var_off */ __update_reg_bounds(dst_reg);}
static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ bool src_known = tnum_subreg_is_const(src_reg->var_off); bool dst_known = tnum_subreg_is_const(dst_reg->var_off); struct tnum var32_off = tnum_subreg(dst_reg->var_off);
if (src_known && dst_known) { __mark_reg32_known(dst_reg, var32_off.value); return; }
/* We get both minimum and maximum from the var32_off. */ reg_set_urange32(dst_reg, var32_off.value, var32_off.value | var32_off.mask);}
static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ bool src_known = tnum_is_const(src_reg->var_off); bool dst_known = tnum_is_const(dst_reg->var_off);
if (src_known && dst_known) { /* dst_reg->var_off.value has been updated earlier */ __mark_reg_known(dst_reg, dst_reg->var_off.value); return; }
/* We get both minimum and maximum from the var_off. */ reg_set_urange64(dst_reg, dst_reg->var_off.value, dst_reg->var_off.value | dst_reg->var_off.mask);}
static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val){ /* If we might shift our top bit out, then we know nothing */ if (umax_val > 31 || reg_u32_max(dst_reg) > 1ULL << (31 - umax_val)) reg_set_urange32(dst_reg, 0, U32_MAX); else /* We lose all sign bit information (except what we can pick * up from var_off) */ reg_set_urange32(dst_reg, reg_u32_min(dst_reg) << umin_val, reg_u32_max(dst_reg) << umax_val);}
static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u32 umax_val = reg_u32_max(src_reg); u32 umin_val = reg_u32_min(src_reg); /* u32 alu operation will zext upper bits */ struct tnum subreg = tnum_subreg(dst_reg->var_off);
__scalar32_min_max_lsh(dst_reg, umin_val, umax_val); dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); /* Not required but being careful mark reg64 bounds as unknown so * that we are forced to pick them up from tnum and zext later and * if some path skips this step we are still safe. */ __mark_reg64_unbounded(dst_reg); __update_reg32_bounds(dst_reg);}
static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val){ struct cnum64 u, s;
/* Special case <<32 because it is a common compiler pattern to sign * extend subreg by doing <<32 s>>32. smin/smax assignments are correct * because s32 bounds don't flip sign when shifting to the left by * 32bits. */ if (umin_val == 32 && umax_val == 32) s = cnum64_from_srange((s64)reg_s32_min(dst_reg) << 32, (s64)reg_s32_max(dst_reg) << 32); else s = CNUM64_UNBOUNDED;
/* If we might shift our top bit out, then we know nothing */ if (reg_umax(dst_reg) > 1ULL << (63 - umax_val)) u = CNUM64_UNBOUNDED; else u = cnum64_from_urange(reg_umin(dst_reg) << umin_val, reg_umax(dst_reg) << umax_val);
dst_reg->r64 = cnum64_intersect(u, s);}
static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u64 umax_val = reg_umax(src_reg); u64 umin_val = reg_umin(src_reg);
/* scalar64 calc uses 32bit unshifted bounds so must be called first */ __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); /* We may learn something more from the var_off */ __update_reg_bounds(dst_reg);}
static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ struct tnum subreg = tnum_subreg(dst_reg->var_off); u32 umax_val = reg_u32_max(src_reg); u32 umin_val = reg_u32_min(src_reg);
/* BPF_RSH is an unsigned shift. If the value in dst_reg might * be negative, then either: * 1) src_reg might be zero, so the sign bit of the result is * unknown, so we lose our signed bounds * 2) it's known negative, thus the unsigned bounds capture the * signed bounds * 3) the signed bounds cross zero, so they tell us nothing * about the result * If the value in dst_reg is known nonnegative, then again the * unsigned bounds capture the signed bounds. * Thus, in all cases it suffices to blow away our signed bounds * and rely on inferring new ones from the unsigned bounds and * var_off of the result. */
dst_reg->var_off = tnum_rshift(subreg, umin_val); reg_set_urange32(dst_reg, reg_u32_min(dst_reg) >> umax_val, reg_u32_max(dst_reg) >> umin_val);
__mark_reg64_unbounded(dst_reg); __update_reg32_bounds(dst_reg);}
static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u64 umax_val = reg_umax(src_reg); u64 umin_val = reg_umin(src_reg);
/* BPF_RSH is an unsigned shift. If the value in dst_reg might * be negative, then either: * 1) src_reg might be zero, so the sign bit of the result is * unknown, so we lose our signed bounds * 2) it's known negative, thus the unsigned bounds capture the * signed bounds * 3) the signed bounds cross zero, so they tell us nothing * about the result * If the value in dst_reg is known nonnegative, then again the * unsigned bounds capture the signed bounds. * Thus, in all cases it suffices to blow away our signed bounds * and rely on inferring new ones from the unsigned bounds and * var_off of the result. */ dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); reg_set_urange64(dst_reg, reg_umin(dst_reg) >> umax_val, reg_umax(dst_reg) >> umin_val);
/* Its not easy to operate on alu32 bounds here because it depends * on bits being shifted in. Take easy way out and mark unbounded * so we can recalculate later from tnum. */ __mark_reg32_unbounded(dst_reg); __update_reg_bounds(dst_reg);}
static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u64 umin_val = reg_u32_min(src_reg);
/* Upon reaching here, src_known is true and * umax_val is equal to umin_val. * Blow away the dst_reg umin_value/umax_value and rely on * dst_reg var_off to refine the result. */ reg_set_srange32(dst_reg, (u32)(((s32)reg_s32_min(dst_reg)) >> umin_val), (u32)(((s32)reg_s32_max(dst_reg)) >> umin_val));
dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
__mark_reg64_unbounded(dst_reg); __update_reg32_bounds(dst_reg);}
static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg){ u64 umin_val = reg_umin(src_reg);
/* Upon reaching here, src_known is true and umax_val is equal * to umin_val. */ reg_set_srange64(dst_reg, reg_smin(dst_reg) >> umin_val, reg_smax(dst_reg) >> umin_val);
dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
/* Its not easy to operate on alu32 bounds here because it depends * on bits being shifted in from upper 32-bits. Take easy way out * and mark unbounded so we can recalculate later from tnum. */ __mark_reg32_unbounded(dst_reg); __update_reg_bounds(dst_reg);}
static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn){ /* * Byte swap operation - update var_off using tnum_bswap. * Three cases: * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE) * unconditional swap * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE) * swap on big-endian, truncation or no-op on little-endian * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE) * swap on little-endian, truncation or no-op on big-endian */
bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64; bool to_le = BPF_SRC(insn->code) == BPF_TO_LE; bool is_big_endian;#ifdef CONFIG_CPU_BIG_ENDIAN is_big_endian = true;#else is_big_endian = false;#endif /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ bool need_bswap = alu64 || (to_le == is_big_endian);
/* * If the register is mutated, manually reset its scalar ID to break * any existing ties and avoid incorrect bounds propagation. */ if (need_bswap || insn->imm == 16 || insn->imm == 32) clear_scalar_id(dst_reg);
if (need_bswap) { if (insn->imm == 16) dst_reg->var_off = tnum_bswap16(dst_reg->var_off); else if (insn->imm == 32) dst_reg->var_off = tnum_bswap32(dst_reg->var_off); else if (insn->imm == 64) dst_reg->var_off = tnum_bswap64(dst_reg->var_off); /* * Byteswap scrambles the range, so we must reset bounds. * Bounds will be re-derived from the new tnum later. */ __mark_reg_unbounded(dst_reg); } /* For bswap16/32, truncate dst register to match the swapped size */ if (insn->imm == 16 || insn->imm == 32) coerce_reg_to_size(dst_reg, insn->imm / 8);}
static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, const struct bpf_reg_state *src_reg){ bool src_is_const = false; u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
if (insn_bitness == 32) { if (tnum_subreg_is_const(src_reg->var_off) && reg_s32_min(src_reg) == reg_s32_max(src_reg) && reg_u32_min(src_reg) == reg_u32_max(src_reg)) src_is_const = true; } else { if (tnum_is_const(src_reg->var_off) && reg_smin(src_reg) == reg_smax(src_reg) && reg_umin(src_reg) == reg_umax(src_reg)) src_is_const = true; }
switch (BPF_OP(insn->code)) { case BPF_ADD: case BPF_SUB: case BPF_NEG: case BPF_AND: case BPF_XOR: case BPF_OR: case BPF_MUL: case BPF_END: return true;
/* * Division and modulo operators range is only safe to compute when the * divisor is a constant. */ case BPF_DIV: case BPF_MOD: return src_is_const;
/* Shift operators range is only computable if shift dimension operand * is a constant. Shifts greater than 31 or 63 are undefined. This * includes shifts by a negative number. */ case BPF_LSH: case BPF_RSH: case BPF_ARSH: return (src_is_const && reg_umax(src_reg) < insn_bitness); default: return false; }}
static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *dst_reg){ struct bpf_verifier_state *branch; struct bpf_reg_state *regs; bool alu32;
if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0) alu32 = false; else if (reg_s32_min(dst_reg) == -1 && reg_s32_max(dst_reg) == 0) alu32 = true; else return 0;
branch = push_stack(env, env->insn_idx, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch);
regs = branch->frame[branch->curframe]->regs; if (alu32) { __mark_reg32_known(®s[insn->dst_reg], 0); __mark_reg32_known(dst_reg, -1ull); } else { __mark_reg_known(®s[insn->dst_reg], 0); __mark_reg_known(dst_reg, -1ull); } return 0;}
/* WARNING: This function does calculations on 64-bit values, but the actual * execution may occur on 32-bit values. Therefore, things like bitshifts * need extra checks in the 32-bit case. */static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *dst_reg, struct bpf_reg_state src_reg){ u8 opcode = BPF_OP(insn->code); s16 off = insn->off; bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); int ret;
if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) { __mark_reg_unknown(env, dst_reg); return 0; }
if (sanitize_needed(opcode)) { ret = sanitize_val_alu(env, insn); if (ret < 0) return sanitize_err(env, insn, ret, NULL, NULL); }
/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. * There are two classes of instructions: The first class we track both * alu32 and alu64 sign/unsigned bounds independently this provides the * greatest amount of precision when alu operations are mixed with jmp32 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, * and BPF_OR. This is possible because these ops have fairly easy to * understand and calculate behavior in both 32-bit and 64-bit alu ops. * See alu32 verifier tests for examples. The second class of * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy * with regards to tracking sign/unsigned bounds because the bits may * cross subreg boundaries in the alu64 case. When this happens we mark * the reg unbounded in the subreg bound space and use the resulting * tnum to calculate an approximation of the sign/unsigned bounds. */ switch (opcode) { case BPF_ADD: scalar32_min_max_add(dst_reg, &src_reg); scalar_min_max_add(dst_reg, &src_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); break; case BPF_SUB: scalar32_min_max_sub(dst_reg, &src_reg); scalar_min_max_sub(dst_reg, &src_reg); dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); break; case BPF_NEG: env->fake_reg[0] = *dst_reg; __mark_reg_known(dst_reg, 0); scalar32_min_max_sub(dst_reg, &env->fake_reg[0]); scalar_min_max_sub(dst_reg, &env->fake_reg[0]); dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off); break; case BPF_MUL: dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); scalar32_min_max_mul(dst_reg, &src_reg); scalar_min_max_mul(dst_reg, &src_reg); break; case BPF_DIV: /* BPF div specification: x / 0 = 0 */ if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) { ___mark_reg_known(dst_reg, 0); break; } if (alu32) if (off == 1) scalar32_min_max_sdiv(dst_reg, &src_reg); else scalar32_min_max_udiv(dst_reg, &src_reg); else if (off == 1) scalar_min_max_sdiv(dst_reg, &src_reg); else scalar_min_max_udiv(dst_reg, &src_reg); break; case BPF_MOD: /* BPF mod specification: x % 0 = x */ if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) break; if (alu32) if (off == 1) scalar32_min_max_smod(dst_reg, &src_reg); else scalar32_min_max_umod(dst_reg, &src_reg); else if (off == 1) scalar_min_max_smod(dst_reg, &src_reg); else scalar_min_max_umod(dst_reg, &src_reg); break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { ret = maybe_fork_scalars(env, insn, dst_reg); if (ret) return ret; } dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { ret = maybe_fork_scalars(env, insn, dst_reg); if (ret) return ret; } dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); break; case BPF_XOR: dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); scalar32_min_max_xor(dst_reg, &src_reg); scalar_min_max_xor(dst_reg, &src_reg); break; case BPF_LSH: if (alu32) scalar32_min_max_lsh(dst_reg, &src_reg); else scalar_min_max_lsh(dst_reg, &src_reg); break; case BPF_RSH: if (alu32) scalar32_min_max_rsh(dst_reg, &src_reg); else scalar_min_max_rsh(dst_reg, &src_reg); break; case BPF_ARSH: if (alu32) scalar32_min_max_arsh(dst_reg, &src_reg); else scalar_min_max_arsh(dst_reg, &src_reg); break; case BPF_END: scalar_byte_swap(dst_reg, insn); break; default: break; }
/* * ALU32 ops are zero extended into 64bit register. * * BPF_END is already handled inside the helper (truncation), * so skip zext here to avoid unexpected zero extension. * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40 * This is a 64bit byte swap operation with alu32==true, * but we should not zero extend the result. */ if (alu32 && opcode != BPF_END) zext_32_to_64(dst_reg); reg_bounds_sync(dst_reg); return 0;}
/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max * and var_off. */static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); u8 opcode = BPF_OP(insn->code); int err;
dst_reg = ®s[insn->dst_reg]; if (BPF_SRC(insn->code) == BPF_X) src_reg = ®s[insn->src_reg]; else src_reg = NULL;
/* Case where at least one operand is an arena. */ if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { struct bpf_insn_aux_data *aux = cur_aux(env);
if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg;
dst_reg->subreg_def = env->insn_idx + 1;
if (BPF_CLASS(insn->code) == BPF_ALU64) /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. */ aux->needs_zext = true;
/* Any arithmetic operations are allowed on arena pointers */ return 0; }
if (dst_reg->type != SCALAR_VALUE) ptr_reg = dst_reg;
if (BPF_SRC(insn->code) == BPF_X) { if (src_reg->type != SCALAR_VALUE) { if (dst_reg->type != SCALAR_VALUE) { /* Combining two pointers by any ALU op yields * an arbitrary scalar. Disallow all math except * pointer subtraction */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { mark_reg_unknown(env, regs, insn->dst_reg); return 0; } verbose(env, "R%d pointer %s pointer prohibited\n", insn->dst_reg, bpf_alu_string[opcode >> 4]); return -EACCES; } else { /* scalar += pointer * This is legal, but we have to reverse our * src/dest handling in computing the range */ err = mark_chain_precision(env, insn->dst_reg); if (err) return err; return adjust_ptr_min_max_vals(env, insn, src_reg, dst_reg); } } else if (ptr_reg) { /* pointer += scalar */ err = mark_chain_precision(env, insn->src_reg); if (err) return err; return adjust_ptr_min_max_vals(env, insn, dst_reg, src_reg); } else if (dst_reg->precise) { /* if dst_reg is precise, src_reg should be precise as well */ err = mark_chain_precision(env, insn->src_reg); if (err) return err; } } else { /* Pretend the src is a reg with a known value, since we only * need to be able to read from this state. */ off_reg.type = SCALAR_VALUE; __mark_reg_known(&off_reg, insn->imm); src_reg = &off_reg; if (ptr_reg) /* pointer += K */ return adjust_ptr_min_max_vals(env, insn, ptr_reg, src_reg); }
/* Got here implies adding two SCALAR_VALUEs */ if (WARN_ON_ONCE(ptr_reg)) { print_verifier_state(env, vstate, vstate->curframe, true); verbose(env, "verifier internal error: unexpected ptr_reg\n"); return -EFAULT; } if (WARN_ON(!src_reg)) { print_verifier_state(env, vstate, vstate->curframe, true); verbose(env, "verifier internal error: no src_reg\n"); return -EFAULT; } /* * For alu32 linked register tracking, we need to check dst_reg's * umax_value before the ALU operation. After adjust_scalar_min_max_vals(), * alu32 ops will have zero-extended the result, making umax_value <= U32_MAX. */ u64 dst_umax = reg_umax(dst_reg);
err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); if (err) return err; /* * Compilers can generate the code * r1 = r2 * r1 += 0x1 * if r2 < 1000 goto ... * use r1 in memory access * So remember constant delta between r2 and r1 and update r1 after * 'if' condition. */ if (env->bpf_capable && (BPF_OP(insn->code) == BPF_ADD || BPF_OP(insn->code) == BPF_SUB) && dst_reg->id && is_reg_const(src_reg, alu32) && !(BPF_SRC(insn->code) == BPF_X && insn->src_reg == insn->dst_reg)) { u64 val = reg_const_value(src_reg, alu32); s32 off;
if (!alu32 && ((s64)val < S32_MIN || (s64)val > S32_MAX)) goto clear_id;
if (alu32 && (dst_umax > U32_MAX)) goto clear_id;
off = (s32)val;
if (BPF_OP(insn->code) == BPF_SUB) { /* Negating S32_MIN would overflow */ if (off == S32_MIN) goto clear_id; off = -off; }
if (dst_reg->id & BPF_ADD_CONST) { /* * If the register already went through rX += val * we cannot accumulate another val into rx->off. */clear_id: clear_scalar_id(dst_reg); } else { if (alu32) dst_reg->id |= BPF_ADD_CONST32; else dst_reg->id |= BPF_ADD_CONST64; dst_reg->delta = off; } } else { /* * Make sure ID is cleared otherwise dst_reg min/max could be * incorrectly propagated into other registers by sync_linked_regs() */ clear_scalar_id(dst_reg); } return 0;}
/* check validity of 32-bit and 64-bit arithmetic operations */static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_reg_state *regs = cur_regs(env); u8 opcode = BPF_OP(insn->code); int err;
if (opcode == BPF_END || opcode == BPF_NEG) { /* check src operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err;
if (is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer arithmetic prohibited\n", insn->dst_reg); return -EACCES; }
/* check dest operand */ if (regs[insn->dst_reg].type == SCALAR_VALUE) { err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); err = err ?: adjust_scalar_min_max_vals(env, insn, ®s[insn->dst_reg], regs[insn->dst_reg]); } else { err = check_reg_arg(env, insn->dst_reg, DST_OP); } if (err) return err;
} else if (opcode == BPF_MOV) {
if (BPF_SRC(insn->code) == BPF_X) { if (insn->off == BPF_ADDR_SPACE_CAST) { if (!env->prog->aux->arena) { verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n"); return -EINVAL; } }
/* check src operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; }
/* check dest operand, mark as required later */ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); if (err) return err;
if (BPF_SRC(insn->code) == BPF_X) { struct bpf_reg_state *src_reg = regs + insn->src_reg; struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
if (BPF_CLASS(insn->code) == BPF_ALU64) { if (insn->imm) { /* off == BPF_ADDR_SPACE_CAST */ mark_reg_unknown(env, regs, insn->dst_reg); if (insn->imm == 1) { /* cast from as(1) to as(0) */ dst_reg->type = PTR_TO_ARENA; /* PTR_TO_ARENA is 32-bit */ dst_reg->subreg_def = env->insn_idx + 1; } } else if (insn->off == 0) { /* case: R1 = R2 * copy register state to dest reg */ assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; dst_reg->subreg_def = DEF_NOT_SUBREG; } else { /* case: R1 = (s8, s16 s32)R2 */ if (is_pointer_value(env, insn->src_reg)) { verbose(env, "R%d sign-extension part of pointer\n", insn->src_reg); return -EACCES; } else if (src_reg->type == SCALAR_VALUE) { bool no_sext;
no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); if (no_sext) assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); coerce_reg_to_size_sx(dst_reg, insn->off >> 3); dst_reg->subreg_def = DEF_NOT_SUBREG; } else { mark_reg_unknown(env, regs, insn->dst_reg); } } } else { /* R1 = (u32) R2 */ if (is_pointer_value(env, insn->src_reg)) { verbose(env, "R%d partial copy of pointer\n", insn->src_reg); return -EACCES; } else if (src_reg->type == SCALAR_VALUE) { if (insn->off == 0) { bool is_src_reg_u32 = get_reg_width(src_reg) <= 32;
if (is_src_reg_u32) assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; /* Make sure ID is cleared if src_reg is not in u32 * range otherwise dst_reg min/max could be incorrectly * propagated into src_reg by sync_linked_regs() */ if (!is_src_reg_u32) clear_scalar_id(dst_reg); dst_reg->subreg_def = env->insn_idx + 1; } else { /* case: W1 = (s8, s16)W2 */ bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1));
if (no_sext) assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); dst_reg->subreg_def = env->insn_idx + 1; coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); } } else { mark_reg_unknown(env, regs, insn->dst_reg); } zext_32_to_64(dst_reg); reg_bounds_sync(dst_reg); } } else { /* case: R = imm * remember the value we stored into this reg */ /* clear any state __mark_reg_known doesn't set */ mark_reg_unknown(env, regs, insn->dst_reg); regs[insn->dst_reg].type = SCALAR_VALUE; if (BPF_CLASS(insn->code) == BPF_ALU64) { __mark_reg_known(regs + insn->dst_reg, insn->imm); } else { __mark_reg_known(regs + insn->dst_reg, (u32)insn->imm); } }
} else { /* all other ALU ops: and, sub, xor, add, ... */
if (BPF_SRC(insn->code) == BPF_X) { /* check src1 operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; }
/* check src2 operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err;
if ((opcode == BPF_MOD || opcode == BPF_DIV) && BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { verbose(env, "div by zero\n"); return -EINVAL; }
if ((opcode == BPF_LSH || opcode == BPF_RSH || opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
if (insn->imm < 0 || insn->imm >= size) { verbose(env, "invalid shift %d\n", insn->imm); return -EINVAL; } }
/* check dest operand */ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); err = err ?: adjust_reg_min_max_vals(env, insn); if (err) return err; }
return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu");}
static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, struct bpf_reg_state *dst_reg, enum bpf_reg_type type, bool range_right_open){ struct bpf_func_state *state; struct bpf_reg_state *reg; int new_range;
if (reg_umax(dst_reg) == 0 && range_right_open) /* This doesn't give us any range */ return;
if (reg_umax(dst_reg) > MAX_PACKET_OFF) /* Risk of overflow. For instance, ptr + (1<<63) may be less * than pkt_end, but that's because it's also less than pkt. */ return;
new_range = reg_umax(dst_reg); if (range_right_open) new_range++;
/* Examples for register markings: * * pkt_data in dst register: * * r2 = r3; * r2 += 8; * if (r2 > pkt_end) goto <handle exception> * <access okay> * * r2 = r3; * r2 += 8; * if (r2 < pkt_end) goto <access okay> * <handle exception> * * Where: * r2 == dst_reg, pkt_end == src_reg * r2=pkt(id=n,off=8,r=0) * r3=pkt(id=n,off=0,r=0) * * pkt_data in src register: * * r2 = r3; * r2 += 8; * if (pkt_end >= r2) goto <access okay> * <handle exception> * * r2 = r3; * r2 += 8; * if (pkt_end <= r2) goto <handle exception> * <access okay> * * Where: * pkt_end == dst_reg, r2 == src_reg * r2=pkt(id=n,off=8,r=0) * r3=pkt(id=n,off=0,r=0) * * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) * and [r3, r3 + 8-1) respectively is safe to access depending on * the check. */
/* If our ids match, then we must have the same max_value. And we * don't care about the other reg's fixed offset, since if it's too big * the range won't allow anything. * reg_umax(dst_reg) is known < MAX_PACKET_OFF, therefore it fits in a u16. */ bpf_for_each_reg_in_vstate(vstate, state, reg, ({ if (reg->type == type && reg->id == dst_reg->id) /* keep the maximum range already checked */ reg->range = max(reg->range, new_range); }));}
static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32);static u8 rev_opcode(u8 opcode);
/* * Learn more information about live branches by simulating refinement on both branches. * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means * that branch is dead. */static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32){ /* Fallthrough (FALSE) branch */ regs_refine_cond_op(&env->false_reg1, &env->false_reg2, rev_opcode(opcode), is_jmp32); reg_bounds_sync(&env->false_reg1); reg_bounds_sync(&env->false_reg2); /* * If there is a range bounds violation in *any* of the abstract values in either * reg_states in the FALSE branch (i.e. reg1, reg2), the FALSE branch must be dead. Only * TRUE branch will be taken. */ if (range_bounds_violation(&env->false_reg1) || range_bounds_violation(&env->false_reg2)) return 1;
/* Jump (TRUE) branch */ regs_refine_cond_op(&env->true_reg1, &env->true_reg2, opcode, is_jmp32); reg_bounds_sync(&env->true_reg1); reg_bounds_sync(&env->true_reg2); /* * If there is a range bounds violation in *any* of the abstract values in either * reg_states in the TRUE branch (i.e. true_reg1, true_reg2), the TRUE branch must be dead. * Only FALSE branch will be taken. */ if (range_bounds_violation(&env->true_reg1) || range_bounds_violation(&env->true_reg2)) return 0;
/* Both branches are possible, we can't determine which one will be taken. */ return -1;}
/* * <reg1> <op> <reg2>, currently assuming reg2 is a constant */static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32){ struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off; struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off; u64 umin1 = is_jmp32 ? (u64)reg_u32_min(reg1) : reg_umin(reg1); u64 umax1 = is_jmp32 ? (u64)reg_u32_max(reg1) : reg_umax(reg1); s64 smin1 = is_jmp32 ? (s64)reg_s32_min(reg1) : reg_smin(reg1); s64 smax1 = is_jmp32 ? (s64)reg_s32_max(reg1) : reg_smax(reg1); u64 umin2 = is_jmp32 ? (u64)reg_u32_min(reg2) : reg_umin(reg2); u64 umax2 = is_jmp32 ? (u64)reg_u32_max(reg2) : reg_umax(reg2); s64 smin2 = is_jmp32 ? (s64)reg_s32_min(reg2) : reg_smin(reg2); s64 smax2 = is_jmp32 ? (s64)reg_s32_max(reg2) : reg_smax(reg2);
if (reg1 == reg2) { switch (opcode) { case BPF_JGE: case BPF_JLE: case BPF_JSGE: case BPF_JSLE: case BPF_JEQ: return 1; case BPF_JGT: case BPF_JLT: case BPF_JSGT: case BPF_JSLT: case BPF_JNE: return 0; case BPF_JSET: if (tnum_is_const(t1)) return t1.value != 0; else return (smin1 <= 0 && smax1 >= 0) ? -1 : 1; default: return -1; } }
switch (opcode) { case BPF_JEQ: /* constants, umin/umax and smin/smax checks would be * redundant in this case because they all should match */ if (tnum_is_const(t1) && tnum_is_const(t2)) return t1.value == t2.value; if (!tnum_overlap(t1, t2)) return 0; /* non-overlapping ranges */ if (umin1 > umax2 || umax1 < umin2) return 0; if (smin1 > smax2 || smax1 < smin2) return 0; if (!is_jmp32) { /* if 64-bit ranges are inconclusive, see if we can * utilize 32-bit subrange knowledge to eliminate * branches that can't be taken a priori */ if (reg_u32_min(reg1) > reg_u32_max(reg2) || reg_u32_max(reg1) < reg_u32_min(reg2)) return 0; if (reg_s32_min(reg1) > reg_s32_max(reg2) || reg_s32_max(reg1) < reg_s32_min(reg2)) return 0; } break; case BPF_JNE: /* constants, umin/umax and smin/smax checks would be * redundant in this case because they all should match */ if (tnum_is_const(t1) && tnum_is_const(t2)) return t1.value != t2.value; if (!tnum_overlap(t1, t2)) return 1; /* non-overlapping ranges */ if (umin1 > umax2 || umax1 < umin2) return 1; if (smin1 > smax2 || smax1 < smin2) return 1; if (!is_jmp32) { /* if 64-bit ranges are inconclusive, see if we can * utilize 32-bit subrange knowledge to eliminate * branches that can't be taken a priori */ if (reg_u32_min(reg1) > reg_u32_max(reg2) || reg_u32_max(reg1) < reg_u32_min(reg2)) return 1; if (reg_s32_min(reg1) > reg_s32_max(reg2) || reg_s32_max(reg1) < reg_s32_min(reg2)) return 1; } break; case BPF_JSET: if (!is_reg_const(reg2, is_jmp32)) { swap(reg1, reg2); swap(t1, t2); } if (!is_reg_const(reg2, is_jmp32)) return -1; if ((~t1.mask & t1.value) & t2.value) return 1; if (!((t1.mask | t1.value) & t2.value)) return 0; break; case BPF_JGT: if (umin1 > umax2) return 1; else if (umax1 <= umin2) return 0; break; case BPF_JSGT: if (smin1 > smax2) return 1; else if (smax1 <= smin2) return 0; break; case BPF_JLT: if (umax1 < umin2) return 1; else if (umin1 >= umax2) return 0; break; case BPF_JSLT: if (smax1 < smin2) return 1; else if (smin1 >= smax2) return 0; break; case BPF_JGE: if (umin1 >= umax2) return 1; else if (umax1 < umin2) return 0; break; case BPF_JSGE: if (smin1 >= smax2) return 1; else if (smax1 < smin2) return 0; break; case BPF_JLE: if (umax1 <= umin2) return 1; else if (umin1 > umax2) return 0; break; case BPF_JSLE: if (smax1 <= smin2) return 1; else if (smin1 > smax2) return 0; break; }
return simulate_both_branches_taken(env, opcode, is_jmp32);}
static int flip_opcode(u32 opcode){ /* How can we transform "a <op> b" into "b <op> a"? */ static const u8 opcode_flip[16] = { /* these stay the same */ [BPF_JEQ >> 4] = BPF_JEQ, [BPF_JNE >> 4] = BPF_JNE, [BPF_JSET >> 4] = BPF_JSET, /* these swap "lesser" and "greater" (L and G in the opcodes) */ [BPF_JGE >> 4] = BPF_JLE, [BPF_JGT >> 4] = BPF_JLT, [BPF_JLE >> 4] = BPF_JGE, [BPF_JLT >> 4] = BPF_JGT, [BPF_JSGE >> 4] = BPF_JSLE, [BPF_JSGT >> 4] = BPF_JSLT, [BPF_JSLE >> 4] = BPF_JSGE, [BPF_JSLT >> 4] = BPF_JSGT }; return opcode_flip[opcode >> 4];}
static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg, u8 opcode){ struct bpf_reg_state *pkt;
if (src_reg->type == PTR_TO_PACKET_END) { pkt = dst_reg; } else if (dst_reg->type == PTR_TO_PACKET_END) { pkt = src_reg; opcode = flip_opcode(opcode); } else { return -1; }
if (pkt->range >= 0) return -1;
switch (opcode) { case BPF_JLE: /* pkt <= pkt_end */ fallthrough; case BPF_JGT: /* pkt > pkt_end */ if (pkt->range == BEYOND_PKT_END) /* pkt has at last one extra byte beyond pkt_end */ return opcode == BPF_JGT; break; case BPF_JLT: /* pkt < pkt_end */ fallthrough; case BPF_JGE: /* pkt >= pkt_end */ if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) return opcode == BPF_JGE; break; } return -1;}
/* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;" * and return: * 1 - branch will be taken and "goto target" will be executed * 0 - branch will not be taken and fall-through to next insn * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value * range [0,10] */static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32){ if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32) return is_pkt_ptr_branch_taken(reg1, reg2, opcode);
if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { u64 val;
/* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { opcode = flip_opcode(opcode); swap(reg1, reg2); } /* and ensure that reg2 is a constant */ if (!is_reg_const(reg2, is_jmp32)) return -1;
if (!reg_not_null(env, reg1)) return -1;
/* If pointer is valid tests against zero will fail so we can * use this to direct branch taken. */ val = reg_const_value(reg2, is_jmp32); if (val != 0) return -1;
switch (opcode) { case BPF_JEQ: return 0; case BPF_JNE: return 1; default: return -1; } }
/* now deal with two scalars, but not necessarily constants */ return is_scalar_branch_taken(env, reg1, reg2, opcode, is_jmp32);}
/* Opcode that corresponds to a *false* branch condition. * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 */static u8 rev_opcode(u8 opcode){ switch (opcode) { case BPF_JEQ: return BPF_JNE; case BPF_JNE: return BPF_JEQ; /* JSET doesn't have it's reverse opcode in BPF, so add * BPF_X flag to denote the reverse of that operation */ case BPF_JSET: return BPF_JSET | BPF_X; case BPF_JSET | BPF_X: return BPF_JSET; case BPF_JGE: return BPF_JLT; case BPF_JGT: return BPF_JLE; case BPF_JLE: return BPF_JGT; case BPF_JLT: return BPF_JGE; case BPF_JSGE: return BPF_JSLT; case BPF_JSGT: return BPF_JSLE; case BPF_JSLE: return BPF_JSGT; case BPF_JSLT: return BPF_JSGE; default: return 0; }}
/* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32){ struct tnum t; u64 val;
/* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */ switch (opcode) { case BPF_JGE: case BPF_JGT: case BPF_JSGE: case BPF_JSGT: opcode = flip_opcode(opcode); swap(reg1, reg2); break; default: break; }
switch (opcode) { case BPF_JEQ: if (is_jmp32) { reg1->r32 = cnum32_intersect(reg1->r32, reg2->r32); reg2->r32 = reg1->r32;
t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); reg2->var_off = tnum_with_subreg(reg2->var_off, t); } else { reg1->r64 = cnum64_intersect(reg1->r64, reg2->r64); reg2->r64 = reg1->r64;
reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off); reg2->var_off = reg1->var_off; } break; case BPF_JNE: if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) break;
/* try to recompute the bound of reg1 if reg2 is a const and * is exactly the edge of reg1. */ val = reg_const_value(reg2, is_jmp32); if (is_jmp32) { /* Complement of the range [val, val] as cnum32. */ cnum32_intersect_with(®1->r32, (struct cnum32){ val + 1, U32_MAX - 1 }); } else { /* Complement of the range [val, val] as cnum64. */ cnum64_intersect_with(®1->r64, (struct cnum64){ val + 1, U64_MAX - 1 }); } break; case BPF_JSET: if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) break; val = reg_const_value(reg2, is_jmp32); /* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X) * requires single bit to learn something useful. E.g., if we * know that `r1 & 0x3` is true, then which bits (0, 1, or both) * are actually set? We can learn something definite only if * it's a single-bit value to begin with. * * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor * bit 1 is set, which we can readily use in adjustments. */ if (!is_power_of_2(val)) break; if (is_jmp32) { t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); } else { reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); } break; case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) break; val = reg_const_value(reg2, is_jmp32); /* Forget the ranges before narrowing tnums, to avoid invariant * violations if we're on a dead branch. */ __mark_reg_unbounded(reg1); if (is_jmp32) { t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); } else { reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val)); } break; case BPF_JLE: if (is_jmp32) { cnum32_intersect_with_urange(®1->r32, 0, reg_u32_max(reg2)); cnum32_intersect_with_urange(®2->r32, reg_u32_min(reg1), U32_MAX); } else { cnum64_intersect_with_urange(®1->r64, 0, reg_umax(reg2)); cnum64_intersect_with_urange(®2->r64, reg_umin(reg1), U64_MAX); } break; case BPF_JLT: if (is_jmp32) { cnum32_intersect_with_urange(®1->r32, 0, reg_u32_max(reg2) - 1); cnum32_intersect_with_urange(®2->r32, reg_u32_min(reg1) + 1, U32_MAX); } else { cnum64_intersect_with_urange(®1->r64, 0, reg_umax(reg2) - 1); cnum64_intersect_with_urange(®2->r64, reg_umin(reg1) + 1, U64_MAX); } break; case BPF_JSLE: if (is_jmp32) { cnum32_intersect_with_srange(®1->r32, S32_MIN, reg_s32_max(reg2)); cnum32_intersect_with_srange(®2->r32, reg_s32_min(reg1), S32_MAX); } else { cnum64_intersect_with_srange(®1->r64, S64_MIN, reg_smax(reg2)); cnum64_intersect_with_srange(®2->r64, reg_smin(reg1), S64_MAX); } break; case BPF_JSLT: if (is_jmp32) { cnum32_intersect_with_srange(®1->r32, S32_MIN, reg_s32_max(reg2) - 1); cnum32_intersect_with_srange(®2->r32, reg_s32_min(reg1) + 1, S32_MAX); } else { cnum64_intersect_with_srange(®1->r64, S64_MIN, reg_smax(reg2) - 1); cnum64_intersect_with_srange(®2->r64, reg_smin(reg1) + 1, S64_MAX); } break; default: return; }}
/* Check for invariant violations on the registers for both branches of a condition */static int regs_bounds_sanity_check_branches(struct bpf_verifier_env *env){ int err;
err = reg_bounds_sanity_check(env, &env->true_reg1, "true_reg1"); err = err ?: reg_bounds_sanity_check(env, &env->true_reg2, "true_reg2"); err = err ?: reg_bounds_sanity_check(env, &env->false_reg1, "false_reg1"); err = err ?: reg_bounds_sanity_check(env, &env->false_reg2, "false_reg2"); return err;}
static void mark_ptr_or_null_reg(struct bpf_func_state *state, struct bpf_reg_state *reg, u32 id, bool is_null){ if (type_may_be_null(reg->type) && reg->id == id && (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { /* Old offset should have been known-zero, because we don't * allow pointer arithmetic on pointers that might be NULL. * If we see this happening, don't convert the register. * * But in some cases, some helpers that return local kptrs * advance offset for the returned pointer. In those cases, * it is fine to expect to see reg->var_off. */ if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) && WARN_ON_ONCE(!tnum_equals_const(reg->var_off, 0))) return; if (is_null) { /* We don't need id from this point * onwards anymore, thus we should better reset it, * so that state pruning has chances to take effect. */ __mark_reg_known_zero(reg); reg->type = SCALAR_VALUE;
return; }
mark_ptr_not_null_reg(reg);
/* * reg->id is preserved for object relationship tracking * and spin_lock lock state tracking */ }}
/* The logic is similar to find_good_pkt_pointers(), both could eventually * be folded together at some point. */static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, bool is_null){ struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = state->regs, *reg; u32 id = regs[regno].id;
if (is_null && find_reference_state(vstate, id)) /* regs[regno] is in the " == NULL" branch. * No one could have freed the reference state before * doing the NULL check. */ WARN_ON_ONCE(release_reference_nomark(vstate, id));
bpf_for_each_reg_in_vstate(vstate, state, reg, ({ mark_ptr_or_null_reg(state, reg, id, is_null); }));}
static bool try_match_pkt_pointers(const struct bpf_insn *insn, struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg, struct bpf_verifier_state *this_branch, struct bpf_verifier_state *other_branch){ if (BPF_SRC(insn->code) != BPF_X) return false;
/* Pointers are always 64-bit. */ if (BPF_CLASS(insn->code) == BPF_JMP32) return false;
switch (BPF_OP(insn->code)) { case BPF_JGT: if ((dst_reg->type == PTR_TO_PACKET && src_reg->type == PTR_TO_PACKET_END) || (dst_reg->type == PTR_TO_PACKET_META && reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ find_good_pkt_pointers(this_branch, dst_reg, dst_reg->type, false); mark_pkt_end(other_branch, insn->dst_reg, true); } else if ((dst_reg->type == PTR_TO_PACKET_END && src_reg->type == PTR_TO_PACKET) || (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && src_reg->type == PTR_TO_PACKET_META)) { /* pkt_end > pkt_data', pkt_data > pkt_meta' */ find_good_pkt_pointers(other_branch, src_reg, src_reg->type, true); mark_pkt_end(this_branch, insn->src_reg, false); } else { return false; } break; case BPF_JLT: if ((dst_reg->type == PTR_TO_PACKET && src_reg->type == PTR_TO_PACKET_END) || (dst_reg->type == PTR_TO_PACKET_META && reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ find_good_pkt_pointers(other_branch, dst_reg, dst_reg->type, true); mark_pkt_end(this_branch, insn->dst_reg, false); } else if ((dst_reg->type == PTR_TO_PACKET_END && src_reg->type == PTR_TO_PACKET) || (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && src_reg->type == PTR_TO_PACKET_META)) { /* pkt_end < pkt_data', pkt_data > pkt_meta' */ find_good_pkt_pointers(this_branch, src_reg, src_reg->type, false); mark_pkt_end(other_branch, insn->src_reg, true); } else { return false; } break; case BPF_JGE: if ((dst_reg->type == PTR_TO_PACKET && src_reg->type == PTR_TO_PACKET_END) || (dst_reg->type == PTR_TO_PACKET_META && reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ find_good_pkt_pointers(this_branch, dst_reg, dst_reg->type, true); mark_pkt_end(other_branch, insn->dst_reg, false); } else if ((dst_reg->type == PTR_TO_PACKET_END && src_reg->type == PTR_TO_PACKET) || (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && src_reg->type == PTR_TO_PACKET_META)) { /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ find_good_pkt_pointers(other_branch, src_reg, src_reg->type, false); mark_pkt_end(this_branch, insn->src_reg, true); } else { return false; } break; case BPF_JLE: if ((dst_reg->type == PTR_TO_PACKET && src_reg->type == PTR_TO_PACKET_END) || (dst_reg->type == PTR_TO_PACKET_META && reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ find_good_pkt_pointers(other_branch, dst_reg, dst_reg->type, false); mark_pkt_end(this_branch, insn->dst_reg, true); } else if ((dst_reg->type == PTR_TO_PACKET_END && src_reg->type == PTR_TO_PACKET) || (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && src_reg->type == PTR_TO_PACKET_META)) { /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ find_good_pkt_pointers(this_branch, src_reg, src_reg->type, true); mark_pkt_end(other_branch, insn->src_reg, false); } else { return false; } break; default: return false; }
return true;}
static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_state *reg, u32 id, u32 frameno, u32 spi_or_reg, bool is_reg){ struct linked_reg *e;
if (reg->type != SCALAR_VALUE || (reg->id & ~BPF_ADD_CONST) != id) return;
e = linked_regs_push(reg_set); if (e) { e->frameno = frameno; e->is_reg = is_reg; e->regno = spi_or_reg; } else { clear_scalar_id(reg); }}
/* For all R being scalar registers or spilled scalar registers * in verifier state, save R in linked_regs if R->id == id. * If there are too many Rs sharing same id, reset id for leftover Rs. */static void collect_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate, u32 id, struct linked_regs *linked_regs){ struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_func_state *func; struct bpf_reg_state *reg; u16 live_regs; int i, j;
id = id & ~BPF_ADD_CONST; for (i = vstate->curframe; i >= 0; i--) { live_regs = aux[bpf_frame_insn_idx(vstate, i)].live_regs_before; func = vstate->frame[i]; for (j = 0; j < BPF_REG_FP; j++) { if (!(live_regs & BIT(j))) continue; reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { if (!bpf_is_spilled_reg(&func->stack[j])) continue; reg = &func->stack[j].spilled_ptr; __collect_linked_regs(linked_regs, reg, id, i, j, false); } }}
/* For all R in linked_regs, copy known_reg range into R * if R->id == known_reg->id. */static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate, struct bpf_reg_state *known_reg, struct linked_regs *linked_regs){ struct bpf_reg_state fake_reg; struct bpf_reg_state *reg; struct linked_reg *e; int i;
for (i = 0; i < linked_regs->cnt; ++i) { e = &linked_regs->entries[i]; reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] : &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr; if (reg->type != SCALAR_VALUE || reg == known_reg) continue; if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) continue; /* * Skip mixed 32/64-bit links: the delta relationship doesn't * hold across different ALU widths. */ if (((reg->id ^ known_reg->id) & BPF_ADD_CONST) == BPF_ADD_CONST) continue; if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || reg->delta == known_reg->delta) { s32 saved_subreg_def = reg->subreg_def;
*reg = *known_reg; reg->subreg_def = saved_subreg_def; } else { s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->delta; u32 saved_id = reg->id;
fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta);
/* reg = known_reg; reg += delta */ *reg = *known_reg; /* * Must preserve off, id and subreg_def flag, * otherwise another sync_linked_regs() will be incorrect. */ reg->delta = saved_off; reg->id = saved_id; reg->subreg_def = saved_subreg_def;
scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); if ((reg->id | known_reg->id) & BPF_ADD_CONST32) zext_32_to_64(reg); reg_bounds_sync(reg); } if (e->is_reg) mark_reg_scratched(env, e->regno); else mark_stack_slot_scratched(env, e->spi); }}
static int check_cond_jmp_op(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx){ struct bpf_verifier_state *this_branch = env->cur_state; struct bpf_verifier_state *other_branch; struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; struct bpf_reg_state *eq_branch_regs; struct linked_regs linked_regs = {}; u8 opcode = BPF_OP(insn->code); int insn_flags = 0; bool is_jmp32; int pred = -1; int err;
/* Only conditional jumps are expected to reach here. */ if (opcode == BPF_JA || opcode > BPF_JCOND) { verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); return -EINVAL; }
if (opcode == BPF_JCOND) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; int idx = *insn_idx;
prev_st = find_prev_entry(env, cur_st->parent, idx);
/* branch out 'fallthrough' insn as a new state to explore */ queued_st = push_stack(env, idx + 1, idx, false); if (IS_ERR(queued_st)) return PTR_ERR(queued_st);
queued_st->may_goto_depth++; if (prev_st) widen_imprecise_scalars(env, prev_st, queued_st); *insn_idx += insn->off; return 0; }
/* check src2 operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err;
dst_reg = ®s[insn->dst_reg]; if (BPF_SRC(insn->code) == BPF_X) { /* check src1 operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err;
src_reg = ®s[insn->src_reg]; if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) && is_pointer_value(env, insn->src_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->src_reg); return -EACCES; }
if (src_reg->type == PTR_TO_STACK) insn_flags |= INSN_F_SRC_REG_STACK; if (dst_reg->type == PTR_TO_STACK) insn_flags |= INSN_F_DST_REG_STACK; } else { src_reg = &env->fake_reg[0]; memset(src_reg, 0, sizeof(*src_reg)); src_reg->type = SCALAR_VALUE; __mark_reg_known(src_reg, insn->imm);
if (dst_reg->type == PTR_TO_STACK) insn_flags |= INSN_F_DST_REG_STACK; }
if (insn_flags) { err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0); if (err) return err; }
is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; env->false_reg1 = *dst_reg; env->false_reg2 = *src_reg; env->true_reg1 = *dst_reg; env->true_reg2 = *src_reg; pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32); if (pred >= 0) { /* If we get here with a dst_reg pointer type it is because * above is_branch_taken() special cased the 0 comparison. */ if (!__is_pointer_value(false, dst_reg)) err = mark_chain_precision(env, insn->dst_reg); if (BPF_SRC(insn->code) == BPF_X && !err && !__is_pointer_value(false, src_reg)) err = mark_chain_precision(env, insn->src_reg); if (err) return err; }
if (pred == 1) { /* Only follow the goto, ignore fall-through. If needed, push * the fall-through branch for simulation under speculative * execution. */ if (!env->bypass_spec_v1) { err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx); if (err < 0) return err; } if (env->log.level & BPF_LOG_LEVEL) print_insn_state(env, this_branch, this_branch->curframe); *insn_idx += insn->off; return 0; } else if (pred == 0) { /* Only follow the fall-through branch, since that's where the * program will go. If needed, push the goto branch for * simulation under speculative execution. */ if (!env->bypass_spec_v1) { err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1, *insn_idx); if (err < 0) return err; } if (env->log.level & BPF_LOG_LEVEL) print_insn_state(env, this_branch, this_branch->curframe); return 0; }
/* Push scalar registers sharing same ID to jump history, * do this before creating 'other_branch', so that both * 'this_branch' and 'other_branch' share this history * if parent state is created. */ if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); if (dst_reg->type == SCALAR_VALUE && dst_reg->id) collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); if (err) return err; }
other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false); if (IS_ERR(other_branch)) return PTR_ERR(other_branch); other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
err = regs_bounds_sanity_check_branches(env); if (err) return err;
*dst_reg = env->false_reg1; *src_reg = env->false_reg2; other_branch_regs[insn->dst_reg] = env->true_reg1; if (BPF_SRC(insn->code) == BPF_X) other_branch_regs[insn->src_reg] = env->true_reg2;
if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id && !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { sync_linked_regs(env, this_branch, src_reg, &linked_regs); sync_linked_regs(env, other_branch, &other_branch_regs[insn->src_reg], &linked_regs); } if (dst_reg->type == SCALAR_VALUE && dst_reg->id && !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { sync_linked_regs(env, this_branch, dst_reg, &linked_regs); sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg], &linked_regs); }
/* if one pointer register is compared to another pointer * register check if PTR_MAYBE_NULL could be lifted. * E.g. register A - maybe null * register B - not null * for JNE A, B, ... - A is not null in the false branch; * for JEQ A, B, ... - A is not null in the true branch. * * Since PTR_TO_BTF_ID points to a kernel struct that does * not need to be null checked by the BPF program, i.e., * could be null even without PTR_MAYBE_NULL marking, so * only propagate nullness when neither reg is that type. */ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && base_type(src_reg->type) != PTR_TO_BTF_ID && base_type(dst_reg->type) != PTR_TO_BTF_ID) { eq_branch_regs = NULL; switch (opcode) { case BPF_JEQ: eq_branch_regs = other_branch_regs; break; case BPF_JNE: eq_branch_regs = regs; break; default: /* do nothing */ break; } if (eq_branch_regs) { if (type_may_be_null(src_reg->type)) mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); else mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); } }
/* detect if R == 0 where R is returned from bpf_map_lookup_elem(). * Also does the same detection for a register whose the value is * known to be 0. * NOTE: these optimizations below are related with pointer comparison * which will never be JMP32. */ if (!is_jmp32 && (opcode == BPF_JEQ || opcode == BPF_JNE) && type_may_be_null(dst_reg->type) && ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) || (BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) { /* Mark all identical registers in each branch as either * safe or unknown depending R == 0 or R != 0 conditional. */ mark_ptr_or_null_regs(this_branch, insn->dst_reg, opcode == BPF_JNE); mark_ptr_or_null_regs(other_branch, insn->dst_reg, opcode == BPF_JEQ); } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], this_branch, other_branch) && is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); return -EACCES; } if (env->log.level & BPF_LOG_LEVEL) print_insn_state(env, this_branch, this_branch->curframe); return 0;}
/* verify BPF_LD_IMM64 instruction */static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_insn_aux_data *aux = cur_aux(env); struct bpf_reg_state *regs = cur_regs(env); struct bpf_reg_state *dst_reg; struct bpf_map *map; int err;
if (BPF_SIZE(insn->code) != BPF_DW) { verbose(env, "invalid BPF_LD_IMM insn\n"); return -EINVAL; }
err = check_reg_arg(env, insn->dst_reg, DST_OP); if (err) return err;
dst_reg = ®s[insn->dst_reg]; if (insn->src_reg == 0) { u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
dst_reg->type = SCALAR_VALUE; __mark_reg_known(®s[insn->dst_reg], imm); return 0; }
/* All special src_reg cases are listed below. From this point onwards * we either succeed and assign a corresponding dst_reg->type after * zeroing the offset, or fail and reject the program. */ mark_reg_known_zero(env, regs, insn->dst_reg);
if (insn->src_reg == BPF_PSEUDO_BTF_ID) { dst_reg->type = aux->btf_var.reg_type; switch (base_type(dst_reg->type)) { case PTR_TO_MEM: dst_reg->mem_size = aux->btf_var.mem_size; break; case PTR_TO_BTF_ID: dst_reg->btf = aux->btf_var.btf; dst_reg->btf_id = aux->btf_var.btf_id; break; default: verifier_bug(env, "pseudo btf id: unexpected dst reg type"); return -EFAULT; } return 0; }
if (insn->src_reg == BPF_PSEUDO_FUNC) { struct bpf_prog_aux *aux = env->prog->aux; u32 subprogno = bpf_find_subprog(env, env->insn_idx + insn->imm + 1);
if (!aux->func_info) { verbose(env, "missing btf func_info\n"); return -EINVAL; } if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { verbose(env, "callback function not static\n"); return -EINVAL; }
dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; return 0; }
map = env->used_maps[aux->map_index];
if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { if (map->map_type == BPF_MAP_TYPE_ARENA) { __mark_reg_unknown(env, dst_reg); dst_reg->map_ptr = map; return 0; } __mark_reg_known(dst_reg, aux->map_off); dst_reg->type = PTR_TO_MAP_VALUE; dst_reg->map_ptr = map; WARN_ON_ONCE(map->map_type != BPF_MAP_TYPE_INSN_ARRAY && map->max_entries != 1); /* We want reg->id to be same (0) as map_value is not distinct */ } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || insn->src_reg == BPF_PSEUDO_MAP_IDX) { dst_reg->type = CONST_PTR_TO_MAP; dst_reg->map_ptr = map; } else { verifier_bug(env, "unexpected src reg value for ldimm64"); return -EFAULT; }
return 0;}
static bool may_access_skb(enum bpf_prog_type type){ switch (type) { case BPF_PROG_TYPE_SOCKET_FILTER: case BPF_PROG_TYPE_SCHED_CLS: case BPF_PROG_TYPE_SCHED_ACT: return true; default: return false; }}
/* verify safety of LD_ABS|LD_IND instructions: * - they can only appear in the programs where ctx == skb * - since they are wrappers of function calls, they scratch R1-R5 registers, * preserve R6-R9, and store return value into R0 * * Implicit input: * ctx == skb == R6 == CTX * * Explicit input: * SRC == any register * IMM == 32-bit immediate * * Output: * R0 - 8/16/32-bit skb data converted to cpu endianness */static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_reg_state *regs = cur_regs(env); static const int ctx_reg = BPF_REG_6; u8 mode = BPF_MODE(insn->code); int i, err;
if (!may_access_skb(resolve_prog_type(env->prog))) { verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); return -EINVAL; }
if (!env->ops->gen_ld_abs) { verifier_bug(env, "gen_ld_abs is null"); return -EFAULT; }
/* check whether implicit source operand (register R6) is readable */ err = check_reg_arg(env, ctx_reg, SRC_OP); if (err) return err;
/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as * gen_ld_abs() may terminate the program at runtime, leading to * reference leak. */ err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]"); if (err) return err;
if (regs[ctx_reg].type != PTR_TO_CTX) { verbose(env, "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); return -EINVAL; }
if (mode == BPF_IND) { /* check explicit source operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; }
err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); if (err < 0) return err;
/* reset caller saved regs to unreadable */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); }
/* mark destination R0 register as readable, since it contains * the value fetched from the packet. * Already marked as written above. */ mark_reg_unknown(env, regs, BPF_REG_0); /* ld_abs load up to 32-bit skb data. */ regs[BPF_REG_0].subreg_def = env->insn_idx + 1; /* * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 * which must be explored by the verifier when in a subprog. */ if (env->cur_state->curframe) { struct bpf_verifier_state *branch;
mark_reg_scratched(env, BPF_REG_0); branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch); mark_reg_known_zero(env, regs, BPF_REG_0); err = prepare_func_exit(env, &env->insn_idx); if (err) return err; env->insn_idx--; } return 0;}
static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range){ enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
/* Default return value range. */ *range = retval_range(0, 1);
switch (prog_type) { case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: switch (env->prog->expected_attach_type) { case BPF_CGROUP_UDP4_RECVMSG: case BPF_CGROUP_UDP6_RECVMSG: case BPF_CGROUP_UNIX_RECVMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: case BPF_CGROUP_UNIX_GETSOCKNAME: *range = retval_range(1, 1); break; case BPF_CGROUP_INET4_BIND: case BPF_CGROUP_INET6_BIND: *range = retval_range(0, 3); break; default: break; } break; case BPF_PROG_TYPE_CGROUP_SKB: if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) *range = retval_range(0, 3); break; case BPF_PROG_TYPE_CGROUP_SOCK: case BPF_PROG_TYPE_SOCK_OPS: case BPF_PROG_TYPE_CGROUP_DEVICE: case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_CGROUP_SOCKOPT: break; case BPF_PROG_TYPE_RAW_TRACEPOINT: if (!env->prog->aux->attach_btf_id) return false; *range = retval_range(0, 0); break; case BPF_PROG_TYPE_TRACING: switch (env->prog->expected_attach_type) { case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: case BPF_TRACE_FSESSION_MULTI: *range = retval_range(0, 0); break; case BPF_TRACE_RAW_TP: case BPF_MODIFY_RETURN: return false; case BPF_TRACE_ITER: default: break; } break; case BPF_PROG_TYPE_KPROBE: switch (env->prog->expected_attach_type) { case BPF_TRACE_KPROBE_SESSION: case BPF_TRACE_UPROBE_SESSION: break; default: return false; } break; case BPF_PROG_TYPE_SK_LOOKUP: *range = retval_range(SK_DROP, SK_PASS); break;
case BPF_PROG_TYPE_LSM: if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { /* no range found, any return value is allowed */ if (!get_func_retval_range(env->prog, range)) return false; /* no restricted range, any return value is allowed */ if (range->minval == S32_MIN && range->maxval == S32_MAX) return false; range->return_32bit = true; } else if (!env->prog->aux->attach_func_proto->type) { /* Make sure programs that attach to void * hooks don't try to modify return value. */ *range = retval_range(1, 1); } break;
case BPF_PROG_TYPE_NETFILTER: *range = retval_range(NF_DROP, NF_ACCEPT); break; case BPF_PROG_TYPE_STRUCT_OPS: *range = retval_range(0, 0); break; case BPF_PROG_TYPE_EXT: /* freplace program can return anything as its return value * depends on the to-be-replaced kernel func or bpf program. */ default: return false; }
/* Continue calculating. */
return true;}
static bool program_returns_void(struct bpf_verifier_env *env){ const struct bpf_prog *prog = env->prog; enum bpf_prog_type prog_type = prog->type;
switch (prog_type) { case BPF_PROG_TYPE_LSM: /* See return_retval_range, for BPF_LSM_CGROUP can be 0 or 0-1 depending on hook. */ if (prog->expected_attach_type != BPF_LSM_CGROUP && !prog->aux->attach_func_proto->type) return true; break; case BPF_PROG_TYPE_STRUCT_OPS: if (!prog->aux->attach_func_proto->type) return true; break; case BPF_PROG_TYPE_EXT: /* * If the actual program is an extension, let it * return void - attaching will succeed only if the * program being replaced also returns void, and since * it has passed verification its actual type doesn't matter. */ if (subprog_returns_void(env, 0)) return true; break; default: break; } return false;}
static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name){ const char *exit_ctx = "At program exit"; struct tnum enforce_attach_type_range = tnum_unknown; const struct bpf_prog *prog = env->prog; struct bpf_reg_state *reg = reg_state(env, regno); struct bpf_retval_range range = retval_range(0, 1); enum bpf_prog_type prog_type = resolve_prog_type(env->prog); struct bpf_func_state *frame = env->cur_state->frame[0]; const struct btf_type *reg_type, *ret_type = NULL; int err;
/* LSM and struct_ops func-ptr's return type could be "void" */ if (!frame->in_async_callback_fn && program_returns_void(env)) return 0;
if (prog_type == BPF_PROG_TYPE_STRUCT_OPS) { /* Allow a struct_ops program to return a referenced kptr if it * matches the operator's return type and is in its unmodified * form. A scalar zero (i.e., a null pointer) is also allowed. */ reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL; ret_type = btf_type_resolve_ptr(prog->aux->attach_btf, prog->aux->attach_func_proto->type, NULL); if (ret_type && ret_type == reg_type && reg_is_referenced(env, reg)) return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false); }
/* eBPF calling convention is such that R0 is used * to return the value from eBPF program. * Make sure that it's readable at this time * of bpf_exit, which means that program wrote * something into it earlier */ err = check_reg_arg(env, regno, SRC_OP); if (err) return err;
if (is_pointer_value(env, regno)) { verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; }
if (frame->in_async_callback_fn) { exit_ctx = "At async callback return"; range = frame->callback_ret_range; goto enforce_retval; }
if (prog_type == BPF_PROG_TYPE_STRUCT_OPS && !ret_type) return 0;
if (prog_type == BPF_PROG_TYPE_CGROUP_SKB && (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS)) enforce_attach_type_range = tnum_range(2, 3);
if (!return_retval_range(env, &range)) return 0;
enforce_retval: if (reg->type != SCALAR_VALUE) { verbose(env, "%s the register R%d is not a known value (%s)\n", exit_ctx, regno, reg_type_str(env, reg->type)); return -EINVAL; }
err = mark_chain_precision(env, regno); if (err) return err;
if (!retval_range_within(range, reg)) { verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name); if (prog->expected_attach_type == BPF_LSM_CGROUP && prog_type == BPF_PROG_TYPE_LSM && !prog->aux->attach_func_proto->type) verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); return -EINVAL; }
if (!tnum_is_unknown(enforce_attach_type_range) && tnum_in(enforce_attach_type_range, reg->var_off)) env->prog->enforce_expected_attach_type = 1; return 0;}
static int check_global_subprog_return_code(struct bpf_verifier_env *env){ struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); struct bpf_func_state *cur_frame = cur_func(env); int err;
if (subprog_returns_void(env, cur_frame->subprogno)) return 0;
err = check_reg_arg(env, BPF_REG_0, SRC_OP); if (err) return err;
/* Pointers to arena are safe to pass between subprograms. */ if (is_arena_reg(env, BPF_REG_0)) return 0;
if (is_pointer_value(env, BPF_REG_0)) { verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); return -EACCES; }
if (reg->type != SCALAR_VALUE) { verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", reg_type_str(env, reg->type)); return -EINVAL; }
return 0;}
/* Bitmask with 1s for all caller saved registers */#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
/* True if do_misc_fixups() replaces calls to helper number 'imm', * replacement patch is presumed to follow bpf_fastcall contract * (see mark_fastcall_pattern_for_call() below). */bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm){ switch (imm) {#ifdef CONFIG_X86_64 case BPF_FUNC_get_smp_processor_id:#ifdef CONFIG_SMP case BPF_FUNC_get_current_task_btf: case BPF_FUNC_get_current_task:#endif return env->prog->jit_requested && bpf_jit_supports_percpu_insn();#endif default: return false; }}
/* If @call is a kfunc or helper call, fills @cs and returns true, * otherwise returns false. */bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs){ struct bpf_kfunc_call_arg_meta meta; const struct bpf_func_proto *fn; int i;
if (bpf_helper_call(call)) {
if (bpf_get_helper_proto(env, call->imm, &fn) < 0) /* error would be reported later */ return false; cs->fastcall = fn->allow_fastcall && (bpf_verifier_inlines_helper_call(env, call->imm) || bpf_jit_inlines_helper_call(call->imm)); cs->is_void = fn->ret_type == RET_VOID; cs->num_params = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { if (fn->arg_type[i] == ARG_DONTCARE) break; cs->num_params++; } return true; }
if (bpf_pseudo_kfunc_call(call)) { int err;
err = bpf_fetch_kfunc_arg_meta(env, call->imm, call->off, &meta); if (err < 0) /* error would be reported later */ return false; cs->num_params = btf_type_vlen(meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; }
return false;}
/* LLVM define a bpf_fastcall function attribute. * This attribute means that function scratches only some of * the caller saved registers defined by ABI. * For BPF the set of such registers could be defined as follows: * - R0 is scratched only if function is non-void; * - R1-R5 are scratched only if corresponding parameter type is defined * in the function prototype. * * The contract between kernel and clang allows to simultaneously use * such functions and maintain backwards compatibility with old * kernels that don't understand bpf_fastcall calls: * * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5 * registers are not scratched by the call; * * - as a post-processing step, clang visits each bpf_fastcall call and adds * spill/fill for every live r0-r5; * * - stack offsets used for the spill/fill are allocated as lowest * stack offsets in whole function and are not used for any other * purposes; * * - when kernel loads a program, it looks for such patterns * (bpf_fastcall function surrounded by spills/fills) and checks if * spill/fill stack offsets are used exclusively in fastcall patterns; * * - if so, and if verifier or current JIT inlines the call to the * bpf_fastcall function (e.g. a helper call), kernel removes unnecessary * spill/fill pairs; * * - when old kernel loads a program, presence of spill/fill pairs * keeps BPF program valid, albeit slightly less efficient. * * For example: * * r1 = 1; * r2 = 2; * *(u64 *)(r10 - 8) = r1; r1 = 1; * *(u64 *)(r10 - 16) = r2; r2 = 2; * call %[to_be_inlined] --> call %[to_be_inlined] * r2 = *(u64 *)(r10 - 16); r0 = r1; * r1 = *(u64 *)(r10 - 8); r0 += r2; * r0 = r1; exit; * r0 += r2; * exit; * * The purpose of mark_fastcall_pattern_for_call is to: * - look for such patterns; * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern; * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction; * - update env->subprog_info[*]->fastcall_stack_off to find an offset * at which bpf_fastcall spill/fill stack slots start; * - update env->subprog_info[*]->keep_fastcall_stack. * * The .fastcall_pattern and .fastcall_stack_off are used by * check_fastcall_stack_contract() to check if every stack access to * fastcall spill/fill stack slot originates from spill/fill * instructions, members of fastcall patterns. * * If such condition holds true for a subprogram, fastcall patterns could * be rewritten by remove_fastcall_spills_fills(). * Otherwise bpf_fastcall patterns are not changed in the subprogram * (code, presumably, generated by an older clang version). * * For example, it is *not* safe to remove spill/fill below: * * r1 = 1; * *(u64 *)(r10 - 8) = r1; r1 = 1; * call %[to_be_inlined] --> call %[to_be_inlined] * r1 = *(u64 *)(r10 - 8); r0 = *(u64 *)(r10 - 8); <---- wrong !!! * r0 = *(u64 *)(r10 - 8); r0 += r1; * r0 += r1; exit; * exit; */static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, struct bpf_subprog_info *subprog, int insn_idx, s16 lowest_off){ struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx; struct bpf_insn *call = &env->prog->insnsi[insn_idx]; u32 clobbered_regs_mask; struct bpf_call_summary cs; u32 expected_regs_mask; s16 off; int i;
if (!bpf_get_call_summary(env, call, &cs)) return;
/* A bitmask specifying which caller saved registers are clobbered * by a call to a helper/kfunc *as if* this helper/kfunc follows * bpf_fastcall contract: * - includes R0 if function is non-void; * - includes R1-R5 if corresponding parameter has is described * in the function prototype. */ clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0); /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;
/* match pairs of form: * * *(u64 *)(r10 - Y) = rX (where Y % 8 == 0) * ... * call %[to_be_inlined] * ... * rX = *(u64 *)(r10 - Y) */ for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) { if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) break; stx = &insns[insn_idx - i]; ldx = &insns[insn_idx + i]; /* must be a stack spill/fill pair */ if (stx->code != (BPF_STX | BPF_MEM | BPF_DW) || ldx->code != (BPF_LDX | BPF_MEM | BPF_DW) || stx->dst_reg != BPF_REG_10 || ldx->src_reg != BPF_REG_10) break; /* must be a spill/fill for the same reg */ if (stx->src_reg != ldx->dst_reg) break; /* must be one of the previously unseen registers */ if ((BIT(stx->src_reg) & expected_regs_mask) == 0) break; /* must be a spill/fill for the same expected offset, * no need to check offset alignment, BPF_DW stack access * is always 8-byte aligned. */ if (stx->off != off || ldx->off != off) break; expected_regs_mask &= ~BIT(stx->src_reg); env->insn_aux_data[insn_idx - i].fastcall_pattern = 1; env->insn_aux_data[insn_idx + i].fastcall_pattern = 1; } if (i == 1) return;
/* Conditionally set 'fastcall_spills_num' to allow forward * compatibility when more helper functions are marked as * bpf_fastcall at compile time than current kernel supports, e.g: * * 1: *(u64 *)(r10 - 8) = r1 * 2: call A ;; assume A is bpf_fastcall for current kernel * 3: r1 = *(u64 *)(r10 - 8) * 4: *(u64 *)(r10 - 8) = r1 * 5: call B ;; assume B is not bpf_fastcall for current kernel * 6: r1 = *(u64 *)(r10 - 8) * * There is no need to block bpf_fastcall rewrite for such program. * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy, * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills() * does not remove spill/fill pair {4,6}. */ if (cs.fastcall) env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1; else subprog->keep_fastcall_stack = 1; subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);}
static int mark_fastcall_patterns(struct bpf_verifier_env *env){ struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_insn *insn; s16 lowest_off; int s, i;
for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { /* find lowest stack spill offset used in this subprog */ lowest_off = 0; for (i = subprog->start; i < (subprog + 1)->start; ++i) { insn = env->prog->insnsi + i; if (insn->code != (BPF_STX | BPF_MEM | BPF_DW) || insn->dst_reg != BPF_REG_10) continue; lowest_off = min(lowest_off, insn->off); } /* use this offset to find fastcall patterns */ for (i = subprog->start; i < (subprog + 1)->start; ++i) { insn = env->prog->insnsi + i; if (insn->code != (BPF_JMP | BPF_CALL)) continue; mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); } } return 0;}
static void adjust_btf_func(struct bpf_verifier_env *env){ struct bpf_prog_aux *aux = env->prog->aux; int i;
if (!aux->func_info) return;
/* func_info is not available for hidden subprogs */ for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++) aux->func_info[i].insn_off = env->subprog_info[i].start;}
/* Find id in idset and increment its count, or add new entry */static void idset_cnt_inc(struct bpf_idset *idset, u32 id){ u32 i;
for (i = 0; i < idset->num_ids; i++) { if (idset->entries[i].id == id) { idset->entries[i].cnt++; return; } } /* New id */ if (idset->num_ids < BPF_ID_MAP_SIZE) { idset->entries[idset->num_ids].id = id; idset->entries[idset->num_ids].cnt = 1; idset->num_ids++; }}
/* Find id in idset and return its count, or 0 if not found */static u32 idset_cnt_get(struct bpf_idset *idset, u32 id){ u32 i;
for (i = 0; i < idset->num_ids; i++) { if (idset->entries[i].id == id) return idset->entries[i].cnt; } return 0;}
/* * Clear singular scalar ids in a state. * A register with a non-zero id is called singular if no other register shares * the same base id. Such registers can be treated as independent (id=0). */void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st){ struct bpf_idset *idset = &env->idset_scratch; struct bpf_func_state *func; struct bpf_reg_state *reg;
idset->num_ids = 0;
bpf_for_each_reg_in_vstate(st, func, reg, ({ if (reg->type != SCALAR_VALUE) continue; if (!reg->id) continue; idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); }));
bpf_for_each_reg_in_vstate(st, func, reg, ({ if (reg->type != SCALAR_VALUE) continue; if (!reg->id) continue; if (idset_cnt_get(idset, reg->id & ~BPF_ADD_CONST) == 1) clear_scalar_id(reg); }));}
/* Return true if it's OK to have the same insn return a different type. */static bool reg_type_mismatch_ok(enum bpf_reg_type type){ switch (base_type(type)) { case PTR_TO_CTX: case PTR_TO_SOCKET: case PTR_TO_SOCK_COMMON: case PTR_TO_TCP_SOCK: case PTR_TO_XDP_SOCK: case PTR_TO_BTF_ID: case PTR_TO_ARENA: return false; default: return true; }}
/* If an instruction was previously used with particular pointer types, then we * need to be careful to avoid cases such as the below, where it may be ok * for one branch accessing the pointer, but not ok for the other branch: * * R1 = sock_ptr * goto X; * ... * R1 = some_other_valid_ptr; * goto X; * ... * R2 = *(u32 *)(R1 + 0); */static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev){ return src != prev && (!reg_type_mismatch_ok(src) || !reg_type_mismatch_ok(prev));}
static bool is_ptr_to_mem_or_btf_id(enum bpf_reg_type type){ switch (base_type(type)) { case PTR_TO_MEM: case PTR_TO_BTF_ID: return true; default: return false; }}
static bool is_ptr_to_mem(enum bpf_reg_type type){ return base_type(type) == PTR_TO_MEM;}
static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch){ enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; enum bpf_reg_type merged_type;
if (*prev_type == NOT_INIT) { /* Saw a valid insn * dst_reg = *(u32 *)(src_reg + off) * save type to validate intersecting paths */ *prev_type = type; } else if (reg_type_mismatch(type, *prev_type)) { /* Abuser program is trying to use the same insn * dst_reg = *(u32*) (src_reg + off) * with different pointer types: * src_reg == ctx in one branch and * src_reg == stack|map in some other branch. * Reject it. */ if (allow_trust_mismatch && is_ptr_to_mem_or_btf_id(type) && is_ptr_to_mem_or_btf_id(*prev_type)) { /* * Have to support a use case when one path through * the program yields TRUSTED pointer while another * is UNTRUSTED. Fallback to UNTRUSTED to generate * BPF_PROBE_MEM/BPF_PROBE_MEMSX. * Same behavior of MEM_RDONLY flag. */ if (is_ptr_to_mem(type) || is_ptr_to_mem(*prev_type)) merged_type = PTR_TO_MEM; else merged_type = PTR_TO_BTF_ID; if ((type & PTR_UNTRUSTED) || (*prev_type & PTR_UNTRUSTED)) merged_type |= PTR_UNTRUSTED; if ((type & MEM_RDONLY) || (*prev_type & MEM_RDONLY)) merged_type |= MEM_RDONLY; *prev_type = merged_type; } else { verbose(env, "same insn cannot be used with different pointers\n"); return -EINVAL; } }
return 0;}
enum { PROCESS_BPF_EXIT = 1, INSN_IDX_UPDATED = 2,};
static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit){ struct bpf_func_state *cur_frame = cur_func(env);
/* We must do check_reference_leak here before * prepare_func_exit to handle the case when * state->curframe > 0, it may be a callback function, * for which reference_state must match caller reference * state when it exits. */ int err = check_resource_leak(env, exception_exit, exception_exit || !env->cur_state->curframe, exception_exit ? "bpf_throw" : "BPF_EXIT instruction in main prog"); if (err) return err;
/* The side effect of the prepare_func_exit which is * being skipped is that it frees bpf_func_state. * Typically, process_bpf_exit will only be hit with * outermost exit. copy_verifier_state in pop_stack will * handle freeing of any extra bpf_func_state left over * from not processing all nested function exits. We * also skip return code checks as they are not needed * for exceptional exits. */ if (exception_exit) return PROCESS_BPF_EXIT;
if (env->cur_state->curframe) { /* exit from nested function */ err = prepare_func_exit(env, &env->insn_idx); if (err) return err; *do_print_state = true; return INSN_IDX_UPDATED; }
/* * Return from a regular global subprogram differs from return * from the main program or async/exception callback. * Main program exit implies return code restrictions * that depend on program type. * Exit from exception callback is equivalent to main program exit. * Exit from async callback implies return code restrictions * that depend on async scheduling mechanism. */ if (cur_frame->subprogno && !cur_frame->in_async_callback_fn && !cur_frame->in_exception_callback_fn) err = check_global_subprog_return_code(env); else err = check_return_code(env, BPF_REG_0, "R0"); if (err) return err; return PROCESS_BPF_EXIT;}
static int indirect_jump_min_max_index(struct bpf_verifier_env *env, int regno, struct bpf_map *map, u32 *pmin_index, u32 *pmax_index){ struct bpf_reg_state *reg = reg_state(env, regno); u64 min_index = reg_umin(reg); u64 max_index = reg_umax(reg); const u32 size = 8;
if (min_index > (u64) U32_MAX * size) { verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg_umin(reg)); return -ERANGE; } if (max_index > (u64) U32_MAX * size) { verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg_umax(reg)); return -ERANGE; }
min_index /= size; max_index /= size;
if (max_index >= map->max_entries) { verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n", regno, min_index, max_index, map->max_entries); return -EINVAL; }
*pmin_index = min_index; *pmax_index = max_index; return 0;}
/* gotox *dst_reg */static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn){ struct bpf_verifier_state *other_branch; struct bpf_reg_state *dst_reg; struct bpf_map *map; u32 min_index, max_index; int err = 0; int n; int i;
dst_reg = reg_state(env, insn->dst_reg); if (dst_reg->type != PTR_TO_INSN) { verbose(env, "R%d has type %s, expected PTR_TO_INSN\n", insn->dst_reg, reg_type_str(env, dst_reg->type)); return -EINVAL; }
map = dst_reg->map_ptr; if (verifier_bug_if(!map, env, "R%d has an empty map pointer", insn->dst_reg)) return -EFAULT;
if (verifier_bug_if(map->map_type != BPF_MAP_TYPE_INSN_ARRAY, env, "R%d has incorrect map type %d", insn->dst_reg, map->map_type)) return -EFAULT;
err = indirect_jump_min_max_index(env, insn->dst_reg, map, &min_index, &max_index); if (err) return err;
/* Ensure that the buffer is large enough */ if (!env->gotox_tmp_buf || env->gotox_tmp_buf->cnt < max_index - min_index + 1) { env->gotox_tmp_buf = bpf_iarray_realloc(env->gotox_tmp_buf, max_index - min_index + 1); if (!env->gotox_tmp_buf) return -ENOMEM; }
n = bpf_copy_insn_array_uniq(map, min_index, max_index, env->gotox_tmp_buf->items); if (n < 0) return n; if (n == 0) { verbose(env, "register R%d doesn't point to any offset in map id=%d\n", insn->dst_reg, map->id); return -EINVAL; }
for (i = 0; i < n - 1; i++) { mark_indirect_target(env, env->gotox_tmp_buf->items[i]); other_branch = push_stack(env, env->gotox_tmp_buf->items[i], env->insn_idx, env->cur_state->speculative); if (IS_ERR(other_branch)) return PTR_ERR(other_branch); } env->insn_idx = env->gotox_tmp_buf->items[n-1]; mark_indirect_target(env, env->insn_idx); return INSN_IDX_UPDATED;}
static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state){ int err; struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx]; u8 class = BPF_CLASS(insn->code);
switch (class) { case BPF_ALU: case BPF_ALU64: return check_alu_op(env, insn);
case BPF_LDX: return check_load_mem(env, insn, false, BPF_MODE(insn->code) == BPF_MEMSX, true, "ldx");
case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) return check_atomic(env, insn); return check_store_reg(env, insn, false);
case BPF_ST: { /* Handle stack arg write (store immediate) */ if (is_stack_arg_st(insn)) { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe];
return check_stack_arg_write(env, state, insn->off, NULL); }
enum bpf_reg_type dst_reg_type;
err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err;
dst_reg_type = cur_regs(env)[insn->dst_reg].type;
err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) return err;
return save_aux_ptr_type(env, dst_reg_type, false); } case BPF_JMP: case BPF_JMP32: { u8 opcode = BPF_OP(insn->code);
env->jmps_processed++; if (opcode == BPF_CALL) { if (env->cur_state->active_locks) { if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock && insn->imm != BPF_FUNC_kptr_xchg) || (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { verbose(env, "function calls are not allowed while holding a lock\n"); return -EINVAL; } } mark_reg_scratched(env, BPF_REG_0); if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) cur_func(env)->no_stack_arg_load = true; if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) return check_kfunc_call(env, insn, &env->insn_idx); return check_helper_call(env, insn, &env->insn_idx); } else if (opcode == BPF_JA) { if (BPF_SRC(insn->code) == BPF_X) return check_indirect_jump(env, insn);
if (class == BPF_JMP) env->insn_idx += insn->off + 1; else env->insn_idx += insn->imm + 1; return INSN_IDX_UPDATED; } else if (opcode == BPF_EXIT) { return process_bpf_exit_full(env, do_print_state, false); } return check_cond_jmp_op(env, insn, &env->insn_idx); } case BPF_LD: { u8 mode = BPF_MODE(insn->code);
if (mode == BPF_ABS || mode == BPF_IND) return check_ld_abs(env, insn);
if (mode == BPF_IMM) { err = check_ld_imm(env, insn); if (err) return err;
env->insn_idx++; sanitize_mark_insn_seen(env); } return 0; } } /* all class values are handled above. silence compiler warning */ return -EFAULT;}
static int do_check(struct bpf_verifier_env *env){ bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_verifier_state *state = env->cur_state; struct bpf_insn *insns = env->prog->insnsi; int insn_cnt = env->prog->len; bool do_print_state = false; int prev_insn_idx = -1;
for (;;) { struct bpf_insn *insn; struct bpf_insn_aux_data *insn_aux; int err;
/* reset current history entry on each new instruction */ env->cur_hist_ent = NULL;
env->prev_insn_idx = prev_insn_idx; if (env->insn_idx >= insn_cnt) { verbose(env, "invalid insn idx %d insn_cnt %d\n", env->insn_idx, insn_cnt); return -EFAULT; }
insn = &insns[env->insn_idx]; insn_aux = &env->insn_aux_data[env->insn_idx];
if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); return -E2BIG; }
state->last_insn_idx = env->prev_insn_idx; state->insn_idx = env->insn_idx;
if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); if (err < 0) return err; if (err == 1) { /* found equivalent state, can prune the search */ if (env->log.level & BPF_LOG_LEVEL) { if (do_print_state) verbose(env, "\nfrom %d to %d%s: safe\n", env->prev_insn_idx, env->insn_idx, env->cur_state->speculative ? " (speculative execution)" : ""); else verbose(env, "%d: safe\n", env->insn_idx); } goto process_bpf_exit; } }
if (bpf_is_jmp_point(env, env->insn_idx)) { err = bpf_push_jmp_history(env, state, 0, 0, 0, 0); if (err) return err; }
if (signal_pending(current)) return -EAGAIN;
if (need_resched()) cond_resched();
if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, env->cur_state->speculative ? " (speculative execution)" : ""); print_verifier_state(env, state, state->curframe, true); do_print_state = false; }
if (env->log.level & BPF_LOG_LEVEL) { if (verifier_state_scratched(env)) print_insn_state(env, state, state->curframe);
verbose_linfo(env, env->insn_idx, "; "); env->prev_log_pos = env->log.end_pos; verbose(env, "%d: ", env->insn_idx); bpf_verbose_insn(env, insn); env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; env->prev_log_pos = env->log.end_pos; }
if (bpf_prog_is_offloaded(env->prog->aux)) { err = bpf_prog_offload_verify_insn(env, env->insn_idx, env->prev_insn_idx); if (err) return err; }
sanitize_mark_insn_seen(env); prev_insn_idx = env->insn_idx;
/* Sanity check: precomputed constants must match verifier state */ if (!state->speculative && insn_aux->const_reg_mask) { struct bpf_reg_state *regs = cur_regs(env); u16 mask = insn_aux->const_reg_mask;
for (int r = 0; r < ARRAY_SIZE(insn_aux->const_reg_vals); r++) { u32 cval = insn_aux->const_reg_vals[r];
if (!(mask & BIT(r))) continue; if (regs[r].type != SCALAR_VALUE) continue; if (!tnum_is_const(regs[r].var_off)) continue; if (verifier_bug_if((u32)regs[r].var_off.value != cval, env, "const R%d: %u != %llu", r, cval, regs[r].var_off.value)) return -EFAULT; } }
/* Reduce verification complexity by stopping speculative path * verification when a nospec is encountered. */ if (state->speculative && insn_aux->nospec) goto process_bpf_exit;
err = do_check_insn(env, &do_print_state); if (error_recoverable_with_nospec(err) && state->speculative) { /* Prevent this speculative path from ever reaching the * insn that would have been unsafe to execute. */ insn_aux->nospec = true; /* If it was an ADD/SUB insn, potentially remove any * markings for alu sanitization. */ insn_aux->alu_state = 0; goto process_bpf_exit; } else if (err < 0) { return err; } else if (err == PROCESS_BPF_EXIT) { goto process_bpf_exit; } else if (err == INSN_IDX_UPDATED) { } else if (err == 0) { env->insn_idx++; }
if (state->speculative && insn_aux->nospec_result) { /* If we are on a path that performed a jump-op, this * may skip a nospec patched-in after the jump. This can * currently never happen because nospec_result is only * used for the write-ops * `*(size*)(dst_reg+off)=src_reg|imm32` and helper * calls. These must never skip the following insn * (i.e., bpf_insn_successors()'s opcode_info.can_jump * is false). Still, add a warning to document this in * case nospec_result is used elsewhere in the future. * * All non-branch instructions have a single * fall-through edge. For these, nospec_result should * already work. */ if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP || BPF_CLASS(insn->code) == BPF_JMP32) && BPF_OP(insn->code) != BPF_CALL, env, "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT;process_bpf_exit: mark_verifier_state_scratched(env); err = bpf_update_branch_counts(env, env->cur_state); if (err) return err; err = pop_stack(env, &prev_insn_idx, &env->insn_idx, pop_log); if (err < 0) { if (err != -ENOENT) return err; break; } else { do_print_state = true; continue; } } }
return 0;}
static int find_btf_percpu_datasec(struct btf *btf){ const struct btf_type *t; const char *tname; int i, n;
/* * Both vmlinux and module each have their own ".data..percpu" * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF * types to look at only module's own BTF types. */ n = btf_nr_types(btf); for (i = btf_named_start_id(btf, true); i < n; i++) { t = btf_type_by_id(btf, i); if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) continue;
tname = btf_name_by_offset(btf, t->name_off); if (!strcmp(tname, ".data..percpu")) return i; }
return -ENOENT;}
/* * Add btf to the env->used_btfs array. If needed, refcount the * corresponding kernel module. To simplify caller's logic * in case of error or if btf was added before the function * decreases the btf refcount. */static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf){ struct btf_mod_pair *btf_mod; int ret = 0; int i;
/* check whether we recorded this BTF (and maybe module) already */ for (i = 0; i < env->used_btf_cnt; i++) if (env->used_btfs[i].btf == btf) goto ret_put;
if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); ret = -E2BIG; goto ret_put; }
btf_mod = &env->used_btfs[env->used_btf_cnt]; btf_mod->btf = btf; btf_mod->module = NULL;
/* if we reference variables from kernel module, bump its refcount */ if (btf_is_module(btf)) { btf_mod->module = btf_try_get_module(btf); if (!btf_mod->module) { ret = -ENXIO; goto ret_put; } }
env->used_btf_cnt++; return 0;
ret_put: /* Either error or this BTF was already added */ btf_put(btf); return ret;}
/* replace pseudo btf_id with kernel symbol address */static int __check_pseudo_btf_id(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *aux, struct btf *btf){ const struct btf_var_secinfo *vsi; const struct btf_type *datasec; const struct btf_type *t; const char *sym_name; bool percpu = false; u32 type, id = insn->imm; s32 datasec_id; u64 addr; int i;
t = btf_type_by_id(btf, id); if (!t) { verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); return -ENOENT; }
if (!btf_type_is_var(t) && !btf_type_is_func(t)) { verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); return -EINVAL; }
sym_name = btf_name_by_offset(btf, t->name_off); addr = kallsyms_lookup_name(sym_name); if (!addr) { verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", sym_name); return -ENOENT; } insn[0].imm = (u32)addr; insn[1].imm = addr >> 32;
if (btf_type_is_func(t)) { aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; aux->btf_var.mem_size = 0; return 0; }
datasec_id = find_btf_percpu_datasec(btf); if (datasec_id > 0) { datasec = btf_type_by_id(btf, datasec_id); for_each_vsi(i, datasec, vsi) { if (vsi->type == id) { percpu = true; break; } } }
type = t->type; t = btf_type_skip_modifiers(btf, type, NULL); if (percpu) { aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; aux->btf_var.btf = btf; aux->btf_var.btf_id = type; } else if (!btf_type_is_struct(t)) { const struct btf_type *ret; const char *tname; u32 tsize;
/* resolve the type size of ksym. */ ret = btf_resolve_size(btf, t, &tsize); if (IS_ERR(ret)) { tname = btf_name_by_offset(btf, t->name_off); verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret)); return -EINVAL; } aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; aux->btf_var.mem_size = tsize; } else { aux->btf_var.reg_type = PTR_TO_BTF_ID; aux->btf_var.btf = btf; aux->btf_var.btf_id = type; }
return 0;}
static int check_pseudo_btf_id(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *aux){ struct btf *btf; int btf_fd; int err;
btf_fd = insn[1].imm; if (btf_fd) { btf = btf_get_by_fd(btf_fd); if (IS_ERR(btf)) { verbose(env, "invalid module BTF object FD specified.\n"); return -EINVAL; } } else { if (!btf_vmlinux) { verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); return -EINVAL; } btf_get(btf_vmlinux); btf = btf_vmlinux; }
err = __check_pseudo_btf_id(env, insn, aux, btf); if (err) { btf_put(btf); return err; }
return __add_used_btf(env, btf);}
static bool is_tracing_prog_type(enum bpf_prog_type type){ switch (type) { case BPF_PROG_TYPE_KPROBE: case BPF_PROG_TYPE_TRACEPOINT: case BPF_PROG_TYPE_PERF_EVENT: case BPF_PROG_TYPE_RAW_TRACEPOINT: case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: return true; default: return false; }}
static bool bpf_map_is_cgroup_storage(struct bpf_map *map){ return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);}
static int check_map_prog_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, struct bpf_prog *prog)
{ enum bpf_prog_type prog_type = resolve_prog_type(prog);
if (map->excl_prog_sha && memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) { verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); return -EACCES; }
if (btf_record_has_field(map->record, BPF_LIST_HEAD) || btf_record_has_field(map->record, BPF_RB_ROOT)) { if (is_tracing_prog_type(prog_type)) { verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); return -EINVAL; } }
if (btf_record_has_field(map->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); return -EINVAL; }
if (is_tracing_prog_type(prog_type)) { verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); return -EINVAL; } }
if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) && !bpf_offload_prog_map_match(prog, map)) { verbose(env, "offload device mismatch between prog and map\n"); return -EINVAL; }
if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { verbose(env, "bpf_struct_ops map cannot be used in prog\n"); return -EINVAL; }
if (prog->sleepable) switch (map->map_type) { case BPF_MAP_TYPE_HASH: case BPF_MAP_TYPE_RHASH: case BPF_MAP_TYPE_LRU_HASH: case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_PERCPU_HASH: case BPF_MAP_TYPE_PERCPU_ARRAY: case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_LPM_TRIE: case BPF_MAP_TYPE_ARRAY_OF_MAPS: case BPF_MAP_TYPE_HASH_OF_MAPS: case BPF_MAP_TYPE_RINGBUF: case BPF_MAP_TYPE_USER_RINGBUF: case BPF_MAP_TYPE_INODE_STORAGE: case BPF_MAP_TYPE_SK_STORAGE: case BPF_MAP_TYPE_TASK_STORAGE: case BPF_MAP_TYPE_CGRP_STORAGE: case BPF_MAP_TYPE_QUEUE: case BPF_MAP_TYPE_STACK: case BPF_MAP_TYPE_ARENA: case BPF_MAP_TYPE_INSN_ARRAY: case BPF_MAP_TYPE_PROG_ARRAY: break; default: verbose(env, "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); return -EINVAL; }
if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) { verbose(env, "only one cgroup storage of each type is allowed\n"); return -EBUSY; }
if (map->map_type == BPF_MAP_TYPE_ARENA) { if (env->prog->aux->arena) { verbose(env, "Only one arena per program\n"); return -EBUSY; } if (!env->allow_ptr_leaks || !env->bpf_capable) { verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); return -EPERM; } if (!env->prog->jit_requested) { verbose(env, "JIT is required to use arena\n"); return -EOPNOTSUPP; } if (!bpf_jit_supports_arena()) { verbose(env, "JIT doesn't support arena\n"); return -EOPNOTSUPP; } env->prog->aux->arena = (void *)map; if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { verbose(env, "arena's user address must be set via map_extra or mmap()\n"); return -EINVAL; } }
return 0;}
static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map){ int i, err;
/* check whether we recorded this map already */ for (i = 0; i < env->used_map_cnt; i++) if (env->used_maps[i] == map) return i;
if (env->used_map_cnt >= MAX_USED_MAPS) { verbose(env, "The total number of maps per program has reached the limit of %u\n", MAX_USED_MAPS); return -E2BIG; }
err = check_map_prog_compatibility(env, map, env->prog); if (err) return err;
if (env->prog->sleepable) atomic64_inc(&map->sleepable_refcnt);
/* hold the map. If the program is rejected by verifier, * the map will be released by release_maps() or it * will be used by the valid program until it's unloaded * and all maps are released in bpf_free_used_maps() */ bpf_map_inc(map);
env->used_maps[env->used_map_cnt++] = map;
if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { err = bpf_insn_array_init(map, env->prog); if (err) { verbose(env, "Failed to properly initialize insn array\n"); return err; } env->insn_array_maps[env->insn_array_map_cnt++] = map; }
return env->used_map_cnt - 1;}
/* Add map behind fd to used maps list, if it's not already there, and return * its index. * Returns <0 on error, or >= 0 index, on success. */static int add_used_map(struct bpf_verifier_env *env, int fd){ struct bpf_map *map; CLASS(fd, f)(fd);
map = __bpf_map_get(f); if (IS_ERR(map)) { verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); return PTR_ERR(map); }
return __add_used_map(env, map);}
static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn){ u8 class = BPF_CLASS(insn->code); u8 opcode = BPF_OP(insn->code);
switch (opcode) { case BPF_NEG: if (BPF_SRC(insn->code) != BPF_K || insn->src_reg != BPF_REG_0 || insn->off != 0 || insn->imm != 0) { verbose(env, "BPF_NEG uses reserved fields\n"); return -EINVAL; } return 0; case BPF_END: if (insn->src_reg != BPF_REG_0 || insn->off != 0 || (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || (class == BPF_ALU64 && BPF_SRC(insn->code) != BPF_TO_LE)) { verbose(env, "BPF_END uses reserved fields\n"); return -EINVAL; } return 0; case BPF_MOV: if (BPF_SRC(insn->code) == BPF_X) { if (class == BPF_ALU) { if ((insn->off != 0 && insn->off != 8 && insn->off != 16) || insn->imm) { verbose(env, "BPF_MOV uses reserved fields\n"); return -EINVAL; } } else if (insn->off == BPF_ADDR_SPACE_CAST) { if (insn->imm != 1 && insn->imm != 1u << 16) { verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n"); return -EINVAL; } } else if ((insn->off != 0 && insn->off != 8 && insn->off != 16 && insn->off != 32) || insn->imm) { verbose(env, "BPF_MOV uses reserved fields\n"); return -EINVAL; } } else if (insn->src_reg != BPF_REG_0 || insn->off != 0) { verbose(env, "BPF_MOV uses reserved fields\n"); return -EINVAL; } return 0; case BPF_ADD: case BPF_SUB: case BPF_AND: case BPF_OR: case BPF_XOR: case BPF_LSH: case BPF_RSH: case BPF_ARSH: case BPF_MUL: case BPF_DIV: case BPF_MOD: if (BPF_SRC(insn->code) == BPF_X) { if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) || (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { verbose(env, "BPF_ALU uses reserved fields\n"); return -EINVAL; } } else if (insn->src_reg != BPF_REG_0 || (insn->off != 0 && insn->off != 1) || (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { verbose(env, "BPF_ALU uses reserved fields\n"); return -EINVAL; } return 0; default: verbose(env, "invalid BPF_ALU opcode %x\n", opcode); return -EINVAL; }}
static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn){ u8 class = BPF_CLASS(insn->code); u8 opcode = BPF_OP(insn->code);
switch (opcode) { case BPF_CALL: if (BPF_SRC(insn->code) != BPF_K || (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) { verbose(env, "BPF_CALL uses reserved fields\n"); return -EINVAL; } return 0; case BPF_JA: if (BPF_SRC(insn->code) == BPF_X) { if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { verbose(env, "BPF_JA|BPF_X uses reserved fields\n"); return -EINVAL; } } else if (insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 || (class == BPF_JMP && insn->imm != 0) || (class == BPF_JMP32 && insn->off != 0)) { verbose(env, "BPF_JA uses reserved fields\n"); return -EINVAL; } return 0; case BPF_EXIT: if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 || insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) { verbose(env, "BPF_EXIT uses reserved fields\n"); return -EINVAL; } return 0; case BPF_JCOND: if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg != BPF_MAY_GOTO || insn->dst_reg || insn->imm) { verbose(env, "invalid may_goto imm %d\n", insn->imm); return -EINVAL; } return 0; default: if (BPF_SRC(insn->code) == BPF_X) { if (insn->imm != 0) { verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); return -EINVAL; } } else if (insn->src_reg != BPF_REG_0) { verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); return -EINVAL; } return 0; }}
static int check_insn_fields(struct bpf_verifier_env *env, struct bpf_insn *insn){ switch (BPF_CLASS(insn->code)) { case BPF_ALU: case BPF_ALU64: return check_alu_fields(env, insn); case BPF_LDX: if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) || insn->imm != 0) { verbose(env, "BPF_LDX uses reserved fields\n"); return -EINVAL; } return 0; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) return 0; if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { verbose(env, "BPF_STX uses reserved fields\n"); return -EINVAL; } return 0; case BPF_ST: if (BPF_MODE(insn->code) != BPF_MEM || insn->src_reg != BPF_REG_0) { verbose(env, "BPF_ST uses reserved fields\n"); return -EINVAL; } return 0; case BPF_JMP: case BPF_JMP32: return check_jmp_fields(env, insn); case BPF_LD: { u8 mode = BPF_MODE(insn->code);
if (mode == BPF_ABS || mode == BPF_IND) { if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || BPF_SIZE(insn->code) == BPF_DW || (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); return -EINVAL; } } else if (mode != BPF_IMM) { verbose(env, "invalid BPF_LD mode\n"); return -EINVAL; } return 0; } default: verbose(env, "unknown insn class %d\n", BPF_CLASS(insn->code)); return -EINVAL; }}
/* * Check that insns are sane and rewrite pseudo imm in ld_imm64 instructions: * * 1. if it accesses map FD, replace it with actual map pointer. * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. * * NOTE: btf_vmlinux is required for converting pseudo btf_id. */static int check_and_resolve_insns(struct bpf_verifier_env *env){ struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; int i, err;
err = bpf_prog_calc_tag(env->prog); if (err) return err;
for (i = 0; i < insn_cnt; i++, insn++) { if (insn->dst_reg >= MAX_BPF_REG && !is_stack_arg_st(insn) && !is_stack_arg_stx(insn)) { verbose(env, "R%d is invalid\n", insn->dst_reg); return -EINVAL; } if (insn->src_reg >= MAX_BPF_REG && !is_stack_arg_ldx(insn)) { verbose(env, "R%d is invalid\n", insn->src_reg); return -EINVAL; } if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { struct bpf_insn_aux_data *aux; struct bpf_map *map; int map_idx; u64 addr; u32 fd;
if (i == insn_cnt - 1 || insn[1].code != 0 || insn[1].dst_reg != 0 || insn[1].src_reg != 0 || insn[1].off != 0) { verbose(env, "invalid bpf_ld_imm64 insn\n"); return -EINVAL; }
if (insn[0].off != 0) { verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); return -EINVAL; }
if (insn[0].src_reg == 0) /* valid generic load 64-bit imm */ goto next_insn;
if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { aux = &env->insn_aux_data[i]; err = check_pseudo_btf_id(env, insn, aux); if (err) return err; goto next_insn; }
if (insn[0].src_reg == BPF_PSEUDO_FUNC) { aux = &env->insn_aux_data[i]; aux->ptr_type = PTR_TO_FUNC; goto next_insn; }
/* In final convert_pseudo_ld_imm64() step, this is * converted into regular 64-bit imm load insn. */ switch (insn[0].src_reg) { case BPF_PSEUDO_MAP_VALUE: case BPF_PSEUDO_MAP_IDX_VALUE: break; case BPF_PSEUDO_MAP_FD: case BPF_PSEUDO_MAP_IDX: if (insn[1].imm == 0) break; fallthrough; default: verbose(env, "unrecognized bpf_ld_imm64 insn\n"); return -EINVAL; }
switch (insn[0].src_reg) { case BPF_PSEUDO_MAP_IDX_VALUE: case BPF_PSEUDO_MAP_IDX: if (bpfptr_is_null(env->fd_array)) { verbose(env, "fd_idx without fd_array is invalid\n"); return -EPROTO; } if (copy_from_bpfptr_offset(&fd, env->fd_array, insn[0].imm * sizeof(fd), sizeof(fd))) return -EFAULT; break; default: fd = insn[0].imm; break; }
map_idx = add_used_map(env, fd); if (map_idx < 0) return map_idx; map = env->used_maps[map_idx];
aux = &env->insn_aux_data[i]; aux->map_index = map_idx;
if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { addr = (unsigned long)map; } else { u32 off = insn[1].imm;
if (!map->ops->map_direct_value_addr) { verbose(env, "no direct value access support for this map type\n"); return -EINVAL; }
err = map->ops->map_direct_value_addr(map, &addr, off); if (err) { verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", map->value_size, off); return err; }
aux->map_off = off; addr += off; }
insn[0].imm = (u32)addr; insn[1].imm = addr >> 32;
next_insn: insn++; i++; continue; }
/* Basic sanity check before we invest more work here. */ if (!bpf_opcode_in_insntable(insn->code)) { verbose(env, "unknown opcode %02x\n", insn->code); return -EINVAL; }
err = check_insn_fields(env, insn); if (err) return err; }
/* now all pseudo BPF_LD_IMM64 instructions load valid * 'struct bpf_map *' into a register instead of user map_fd. * These pointers will be used later by verifier to validate map access. */ return 0;}
/* drop refcnt of maps used by the rejected program */static void release_maps(struct bpf_verifier_env *env){ __bpf_free_used_maps(env->prog->aux, env->used_maps, env->used_map_cnt);}
/* drop refcnt of maps used by the rejected program */static void release_btfs(struct bpf_verifier_env *env){ __bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt);}
/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env){ struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; int i;
for (i = 0; i < insn_cnt; i++, insn++) { if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) continue; if (insn->src_reg == BPF_PSEUDO_FUNC) continue; insn->src_reg = 0; }}
static void release_insn_arrays(struct bpf_verifier_env *env){ int i;
for (i = 0; i < env->insn_array_map_cnt; i++) bpf_insn_array_release(env->insn_array_maps[i]);}
/* The verifier does more data flow analysis than llvm and will not * explore branches that are dead at run time. Malicious programs can * have dead code too. Therefore replace all dead at-run-time code * with 'ja -1'. * * Just nops are not optimal, e.g. if they would sit at the end of the * program and through another bug we would manage to jump there, then * we'd execute beyond program memory otherwise. Returning exception * code also wouldn't work since we can have subprogs where the dead * code could be located. */static void sanitize_dead_code(struct bpf_verifier_env *env){ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); struct bpf_insn *insn = env->prog->insnsi; const int insn_cnt = env->prog->len; int i;
for (i = 0; i < insn_cnt; i++) { if (aux_data[i].seen) continue; memcpy(insn + i, &trap, sizeof(trap)); aux_data[i].zext_dst = false; }}
static void free_states(struct bpf_verifier_env *env){ struct bpf_verifier_state_list *sl; struct list_head *head, *pos, *tmp; struct bpf_scc_info *info; int i, j;
bpf_free_verifier_state(env->cur_state, true); env->cur_state = NULL; while (!pop_stack(env, NULL, NULL, false));
list_for_each_safe(pos, tmp, &env->free_list) { sl = container_of(pos, struct bpf_verifier_state_list, node); bpf_free_verifier_state(&sl->state, false); kfree(sl); } INIT_LIST_HEAD(&env->free_list);
for (i = 0; i < env->scc_cnt; ++i) { info = env->scc_info[i]; if (!info) continue; for (j = 0; j < info->num_visits; j++) bpf_free_backedges(&info->visits[j]); kvfree(info); env->scc_info[i] = NULL; }
if (!env->explored_states) return;
for (i = 0; i < state_htab_size(env); i++) { head = &env->explored_states[i];
list_for_each_safe(pos, tmp, head) { sl = container_of(pos, struct bpf_verifier_state_list, node); bpf_free_verifier_state(&sl->state, false); kfree(sl); } INIT_LIST_HEAD(&env->explored_states[i]); }}
static int do_check_common(struct bpf_verifier_env *env, int subprog){ bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; int ret, i;
env->prev_linfo = NULL; env->pass_cnt++;
state = kzalloc_obj(struct bpf_verifier_state, GFP_KERNEL_ACCOUNT); if (!state) return -ENOMEM; state->curframe = 0; state->speculative = false; state->branches = 1; state->in_sleepable = env->prog->sleepable; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); return -ENOMEM; } env->cur_state = state; init_func_state(env, state->frame[0], BPF_MAIN_FUNC /* callsite */, 0 /* frameno */, subprog); state->first_insn_idx = env->subprog_info[subprog].start; state->last_insn_idx = -1;
regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { const char *sub_name = subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg;
if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); ret = btf_prepare_func_args(env, subprog); if (ret) goto out;
if (subprog_is_exc_cb(env, subprog)) { state->frame[0]->in_exception_callback_fn = true;
/* * Global functions are scalar or void, make sure * we return a scalar. */ if (subprog_returns_void(env, subprog)) { verbose(env, "exception cb cannot return void\n"); ret = -EINVAL; goto out; }
/* Also ensure the callback only has a single scalar argument. */ if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i];
if (arg->arg_type == ARG_PTR_TO_CTX) { reg->type = PTR_TO_CTX; mark_reg_known_zero(env, regs, i); } else if (arg->arg_type == ARG_ANYTHING) { reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { /* assume unspecial LOCAL dynptr type */ __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { reg->type = PTR_TO_MEM; reg->type |= arg->arg_type & (PTR_MAYBE_NULL | PTR_UNTRUSTED | MEM_RDONLY); mark_reg_known_zero(env, regs, i); reg->mem_size = arg->mem_size; if (arg->arg_type & PTR_MAYBE_NULL) reg->id = ++env->id_gen; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { reg->type = PTR_TO_BTF_ID; if (arg->arg_type & PTR_MAYBE_NULL) reg->type |= PTR_MAYBE_NULL; if (arg->arg_type & PTR_UNTRUSTED) reg->type |= PTR_UNTRUSTED; if (arg->arg_type & PTR_TRUSTED) reg->type |= PTR_TRUSTED; mark_reg_known_zero(env, regs, i); reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */ reg->btf_id = arg->btf_id; reg->id = ++env->id_gen; } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { /* caller can pass either PTR_TO_ARENA or SCALAR */ mark_reg_unknown(env, regs, i); } else { verifier_bug(env, "unhandled arg#%d type %d", i - BPF_REG_1 + 1, arg->arg_type); ret = -EFAULT; goto out; } } if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { verbose(env, "freplace programs with >%d args not supported yet\n", MAX_BPF_FUNC_REG_ARGS); ret = -EINVAL; goto out; } } else { /* if main BPF program has associated BTF info, validate that * it's matching expected signature, and otherwise mark BTF * info for main program as unreliable */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; sub->arg_cnt = 1; sub->stack_arg_cnt = 0; } }
/* 1st arg to a function */ regs[BPF_REG_1].type = PTR_TO_CTX; mark_reg_known_zero(env, regs, BPF_REG_1); }
/* Acquire references for struct_ops program arguments tagged with "__ref" */ if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { for (i = 0; i < aux->ctx_arg_info_size; i++) { ret = aux->ctx_arg_info[i].refcounted ? acquire_reference(env, 0, 0) : 0; if (ret < 0) goto out;
aux->ctx_arg_info[i].ref_id = ret; } }
ret = do_check(env);out: if (!ret && pop_log) bpf_vlog_reset(&env->log, 0); free_states(env); return ret;}
/* Lazily verify all global functions based on their BTF, if they are called * from main BPF program or any of subprograms transitively. * BPF global subprogs called from dead code are not validated. * All callable global functions must pass verification. * Otherwise the whole program is rejected. * Consider: * int bar(int); * int foo(int f) * { * return bar(f); * } * int bar(int b) * { * ... * } * foo() will be verified first for R1=any_scalar_value. During verification it * will be assumed that bar() already verified successfully and call to bar() * from foo() will be checked for type match only. Later bar() will be verified * independently to check that it's safe for R1=any_scalar_value. */static int do_check_subprogs(struct bpf_verifier_env *env){ struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; int i, ret, new_cnt; u32 insn_processed;
if (!aux->func_info) return 0;
/* exception callback is presumed to be always called */ if (env->exception_callback_subprog) subprog_aux(env, env->exception_callback_subprog)->called = true;
again: new_cnt = 0; for (i = 1; i < env->subprog_cnt; i++) { if (!bpf_subprog_is_global(env, i)) continue;
insn_processed = env->insn_processed;
sub_aux = subprog_aux(env, i); if (!sub_aux->called || sub_aux->verified) continue;
env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i); env->subprog_info[i].insn_processed = env->insn_processed - insn_processed; if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", i, subprog_name(env, i)); }
/* We verified new global subprog, it might have called some * more global subprogs that we haven't verified yet, so we * need to do another pass over subprogs to verify those. */ sub_aux->verified = true; new_cnt++; }
/* We can't loop forever as we verify at least one global subprog on * each pass. */ if (new_cnt) goto again;
return 0;}
static int do_check_main(struct bpf_verifier_env *env){ u32 insn_processed = env->insn_processed; int ret;
env->insn_idx = 0; ret = do_check_common(env, 0); env->subprog_info[0].insn_processed = env->insn_processed - insn_processed; if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret;}
static void print_verification_stats(struct bpf_verifier_env *env){ /* Skip over hidden subprogs which are not verified. */ int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
if (env->log.level & BPF_LOG_STATS) { verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, 1000)); verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); for (i = 1; i < subprog_cnt; i++) verbose(env, "+%d", env->subprog_info[i].stack_depth); verbose(env, " max %d\n", env->max_stack_depth); verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); for (i = 1; i < subprog_cnt; i++) if (bpf_subprog_is_global(env, i)) verbose(env, "+%d", env->subprog_info[i].insn_processed); verbose(env, "\n"); } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " "total_states %d peak_states %d mark_read %d\n", env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, env->max_states_per_insn, env->total_states, env->peak_states, env->longest_mark_read_walk);}
int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog, const struct bpf_ctx_arg_aux *info, u32 cnt){ prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL_ACCOUNT); prog->aux->ctx_arg_info_size = cnt;
return prog->aux->ctx_arg_info ? 0 : -ENOMEM;}
static int check_struct_ops_btf_id(struct bpf_verifier_env *env){ const struct btf_type *t, *func_proto; const struct bpf_struct_ops_desc *st_ops_desc; const struct bpf_struct_ops *st_ops; const struct btf_member *member; struct bpf_prog *prog = env->prog; bool has_refcounted_arg = false; u32 btf_id, member_idx, member_off; struct btf *btf; const char *mname; int i, err;
if (!prog->gpl_compatible) { verbose(env, "struct ops programs must have a GPL compatible license\n"); return -EINVAL; }
if (!prog->aux->attach_btf_id) return -ENOTSUPP;
btf = prog->aux->attach_btf; if (btf_is_module(btf)) { /* Make sure st_ops is valid through the lifetime of env */ env->attach_btf_mod = btf_try_get_module(btf); if (!env->attach_btf_mod) { verbose(env, "struct_ops module %s is not found\n", btf_get_name(btf)); return -ENOTSUPP; } }
btf_id = prog->aux->attach_btf_id; st_ops_desc = bpf_struct_ops_find(btf, btf_id); if (!st_ops_desc) { verbose(env, "attach_btf_id %u is not a supported struct\n", btf_id); return -ENOTSUPP; } st_ops = st_ops_desc->st_ops;
t = st_ops_desc->type; member_idx = prog->expected_attach_type; if (member_idx >= btf_type_vlen(t)) { verbose(env, "attach to invalid member idx %u of struct %s\n", member_idx, st_ops->name); return -EINVAL; }
member = &btf_type_member(t)[member_idx]; mname = btf_name_by_offset(btf, member->name_off); func_proto = btf_type_resolve_func_ptr(btf, member->type, NULL); if (!func_proto) { verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", mname, member_idx, st_ops->name); return -EINVAL; }
member_off = __btf_member_bit_offset(t, member) / 8; err = bpf_struct_ops_supported(st_ops, member_off); if (err) { verbose(env, "attach to unsupported member %s of struct %s\n", mname, st_ops->name); return err; }
if (st_ops->check_member) { err = st_ops->check_member(t, member, prog);
if (err) { verbose(env, "attach to unsupported member %s of struct %s\n", mname, st_ops->name); return err; } }
if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) { verbose(env, "Private stack not supported by jit\n"); return -EACCES; }
for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) { if (st_ops_desc->arg_info[member_idx].info[i].refcounted) { has_refcounted_arg = true; break; } }
/* Tail call is not allowed for programs with refcounted arguments since we * cannot guarantee that valid refcounted kptrs will be passed to the callee. */ for (i = 0; i < env->subprog_cnt; i++) { if (has_refcounted_arg && env->subprog_info[i].has_tail_call) { verbose(env, "program with __ref argument cannot tail call\n"); return -EINVAL; } }
prog->aux->st_ops = st_ops; prog->aux->attach_st_ops_member_off = member_off;
prog->aux->attach_func_proto = func_proto; prog->aux->attach_func_name = mname; env->ops = st_ops->verifier_ops;
return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info, st_ops_desc->arg_info[member_idx].cnt);}#define SECURITY_PREFIX "security_"
#ifdef CONFIG_FUNCTION_ERROR_INJECTION
/* list of non-sleepable functions that are otherwise on * ALLOW_ERROR_INJECTION list */BTF_SET_START(btf_non_sleepable_error_inject)/* Three functions below can be called from sleepable and non-sleepable context. * Assume non-sleepable from bpf safety point of view. */BTF_ID(func, __filemap_add_folio)#ifdef CONFIG_FAIL_PAGE_ALLOCBTF_ID(func, should_fail_alloc_page)#endif#ifdef CONFIG_FAILSLABBTF_ID(func, should_failslab)#endifBTF_SET_END(btf_non_sleepable_error_inject)
static int check_non_sleepable_error_inject(u32 btf_id){ return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);}
static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name){ /* fentry/fexit/fmod_ret progs can be sleepable if they are * attached to ALLOW_ERROR_INJECTION and are not in denylist. */ if (!check_non_sleepable_error_inject(btf_id) && within_error_injection_list(addr)) return 0;
return -EINVAL;}
static int check_attach_modify_return(unsigned long addr, const char *func_name){ if (within_error_injection_list(addr) || !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) return 0;
return -EINVAL;}
#else
/* Unfortunately, the arch-specific prefixes are hard-coded in arch syscall code * so we need to hard-code them, too. Ftrace has arch_syscall_match_sym_name() * but that just compares two concrete function names. */static bool has_arch_syscall_prefix(const char *func_name){#if defined(__x86_64__) return !strncmp(func_name, "__x64_", 6);#elif defined(__i386__) return !strncmp(func_name, "__ia32_", 7);#elif defined(__s390x__) return !strncmp(func_name, "__s390x_", 8);#elif defined(__aarch64__) return !strncmp(func_name, "__arm64_", 8);#elif defined(__riscv) return !strncmp(func_name, "__riscv_", 8);#elif defined(__powerpc__) || defined(__powerpc64__) return !strncmp(func_name, "sys_", 4);#elif defined(__loongarch__) return !strncmp(func_name, "sys_", 4);#else return false;#endif}
/* Without error injection, allow sleepable and fmod_ret progs on syscalls. */
static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name){ if (has_arch_syscall_prefix(func_name)) return 0;
return -EINVAL;}
static int check_attach_modify_return(unsigned long addr, const char *func_name){ if (has_arch_syscall_prefix(func_name) || !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) return 0;
return -EINVAL;}
#endif /* CONFIG_FUNCTION_ERROR_INJECTION */
static bool is_tracing_multi_id(const struct bpf_prog *prog, u32 btf_id){ return is_tracing_multi(prog->expected_attach_type) && bpf_multi_func_btf_id[0] == btf_id;}
static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct bpf_prog *prog, const struct btf *btf){ const struct btf_type *t; const char *tname;
switch (prog->type) { case BPF_PROG_TYPE_TRACING: t = btf_type_by_id(btf, btf_id); if (!t) return -EINVAL; tname = btf_name_by_offset(btf, t->name_off); if (!tname) return -EINVAL;
/* * *.multi sleepable programs will pass initial sleepable check, * the actual attached btf ids are checked later during the link * attachment. */ if (is_tracing_multi_id(prog, btf_id)) return 0; if (!check_attach_sleepable(btf_id, addr, tname)) return 0; /* * fentry/fexit/fmod_ret progs can also be sleepable if they are * in the fmodret id set with the KF_SLEEPABLE flag. */ else { u32 *flags = btf_kfunc_is_modify_return(btf, btf_id, prog);
if (flags && (*flags & KF_SLEEPABLE)) return 0; } break; case BPF_PROG_TYPE_LSM: /* * LSM progs check that they are attached to bpf_lsm_*() funcs. * Only some of them are sleepable. */ if (bpf_lsm_is_sleepable_hook(btf_id)) return 0; break; default: break; } return -EINVAL;}
/* * Resolve the prototype describing a trace target's real ABI. A * KF_IMPLICIT_ARGS kfunc has its injected args stripped from the public * prototype, so use the _impl prototype; other targets use their own. */static const struct btf_type *btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id){ const struct btf_type *func; struct module *mod = NULL; const char *name; int implicit;
func = btf_type_by_id(btf, func_id); if (!func || !btf_type_is_func(func)) return NULL; name = btf_name_by_offset(btf, func->name_off);
/* * btf_kfunc_check_flag() reads kfunc_set_tab, which for a module is * stable only once it is live; hold a module ref across the read to * exclude a concurrent module load. */ if (btf_is_module(btf)) { mod = btf_try_get_module(btf); if (!mod) return NULL; } implicit = btf_kfunc_check_flag(btf, func_id, KF_IMPLICIT_ARGS); module_put(mod);
if (implicit == -EINVAL) { bpf_log(log, "kfunc %s has inconsistent KF_IMPLICIT_ARGS\n", name); return NULL; } if (implicit > 0) return find_kfunc_impl_proto(log, btf, name);
return btf_type_by_id(btf, func->type);}
int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info){ bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING; char trace_symbol[KSYM_SYMBOL_LEN]; const char prefix[] = "btf_trace_"; struct bpf_raw_event_map *btp; int ret = 0, subprog = -1, i; const struct btf_type *t; bool conservative = true; const char *tname, *fname; struct btf *btf; long addr = 0; struct module *mod = NULL;
if (!btf_id) { bpf_log(log, "Tracing programs must provide btf_id\n"); return -EINVAL; } btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; if (!btf) { bpf_log(log, "Tracing program can only be attached to another program annotated with BTF\n"); return -EINVAL; } t = btf_type_by_id(btf, btf_id); if (!t) { bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); return -EINVAL; } tname = btf_name_by_offset(btf, t->name_off); if (!tname) { bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); return -EINVAL; } if (tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; bool tgt_changes_pkt_data; bool tgt_might_sleep;
if (bpf_prog_is_dev_bound(prog->aux) && !bpf_prog_dev_bound_match(prog, tgt_prog)) { bpf_log(log, "Target program bound device mismatch"); return -EINVAL; }
for (i = 0; i < aux->func_info_cnt; i++) if (aux->func_info[i].type_id == btf_id) { subprog = i; break; } if (subprog == -1) { bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; } if (aux->func && aux->func[subprog]->aux->exception_cb) { bpf_log(log, "%s programs cannot attach to exception callback\n", prog_extension ? "Extension" : "Tracing"); return -EINVAL; } conservative = aux->func_info_aux[subprog].unreliable; if (prog_extension) { if (conservative) { bpf_log(log, "Cannot replace static functions\n"); return -EINVAL; } if (!prog->jit_requested) { bpf_log(log, "Extension programs should be JITed\n"); return -EINVAL; } tgt_changes_pkt_data = aux->func ? aux->func[subprog]->aux->changes_pkt_data : aux->changes_pkt_data; if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) { bpf_log(log, "Extension program changes packet data, while original does not\n"); return -EINVAL; }
tgt_might_sleep = aux->func ? aux->func[subprog]->aux->might_sleep : aux->might_sleep; if (prog->aux->might_sleep && !tgt_might_sleep) { bpf_log(log, "Extension program may sleep, while original does not\n"); return -EINVAL; } } if (!tgt_prog->jited) { bpf_log(log, "Can attach to only JITed progs\n"); return -EINVAL; } if (prog_tracing) { if (aux->attach_tracing_prog) { /* * Target program is an fentry/fexit which is already attached * to another tracing program. More levels of nesting * attachment are not allowed. */ bpf_log(log, "Cannot nest tracing program attach more than once\n"); return -EINVAL; } } else if (tgt_prog->type == prog->type) { /* * To avoid potential call chain cycles, prevent attaching of a * program extension to another extension. It's ok to attach * fentry/fexit to extension program. */ bpf_log(log, "Cannot recursively attach\n"); return -EINVAL; } if (tgt_prog->type == BPF_PROG_TYPE_TRACING && prog_extension && (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || tgt_prog->expected_attach_type == BPF_TRACE_FEXIT || tgt_prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI || tgt_prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || tgt_prog->expected_attach_type == BPF_TRACE_FSESSION || tgt_prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { /* Program extensions can extend all program types * except fentry/fexit. The reason is the following. * The fentry/fexit programs are used for performance * analysis, stats and can be attached to any program * type. When extension program is replacing XDP function * it is necessary to allow performance analysis of all * functions. Both original XDP program and its program * extension. Hence attaching fentry/fexit to * BPF_PROG_TYPE_EXT is allowed. If extending of * fentry/fexit was allowed it would be possible to create * long call chain fentry->extension->fentry->extension * beyond reasonable stack size. Hence extending fentry * is not allowed. */ bpf_log(log, "Cannot extend fentry/fexit/fsession\n"); return -EINVAL; } } else { if (prog_extension) { bpf_log(log, "Cannot replace kernel functions\n"); return -EINVAL; } }
switch (prog->expected_attach_type) { case BPF_TRACE_RAW_TP: if (tgt_prog) { bpf_log(log, "Only FENTRY/FEXIT/FSESSION progs are attachable to another BPF prog\n"); return -EINVAL; } if (!btf_type_is_typedef(t)) { bpf_log(log, "attach_btf_id %u is not a typedef\n", btf_id); return -EINVAL; } if (strncmp(prefix, tname, sizeof(prefix) - 1)) { bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", btf_id, tname); return -EINVAL; } tname += sizeof(prefix) - 1;
/* The func_proto of "btf_trace_##tname" is generated from typedef without argument * names. Thus using bpf_raw_event_map to get argument names. */ btp = bpf_get_raw_tracepoint(tname); if (!btp) return -EINVAL; if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) { bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n", tname); bpf_put_raw_tracepoint(btp); return -EINVAL; } fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL, trace_symbol); bpf_put_raw_tracepoint(btp);
if (fname) ret = btf_find_by_name_kind(btf, fname, BTF_KIND_FUNC);
if (!fname || ret < 0) { bpf_log(log, "Cannot find btf of tracepoint template, fall back to %s%s.\n", prefix, tname); t = btf_type_by_id(btf, t->type); if (!btf_type_is_ptr(t)) /* should never happen in valid vmlinux build */ return -EINVAL; } else { t = btf_type_by_id(btf, ret); if (!btf_type_is_func(t)) /* should never happen in valid vmlinux build */ return -EINVAL; }
t = btf_type_by_id(btf, t->type); if (!btf_type_is_func_proto(t)) /* should never happen in valid vmlinux build */ return -EINVAL;
break; case BPF_TRACE_ITER: if (!btf_type_is_func(t)) { bpf_log(log, "attach_btf_id %u is not a function\n", btf_id); return -EINVAL; } t = btf_type_by_id(btf, t->type); if (!btf_type_is_func_proto(t)) return -EINVAL; ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret) return ret; break; default: if (!prog_extension) return -EINVAL; fallthrough; case BPF_MODIFY_RETURN: case BPF_LSM_MAC: case BPF_LSM_CGROUP: case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_TRACE_FSESSION: case BPF_TRACE_FSESSION_MULTI: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: if ((prog->expected_attach_type == BPF_TRACE_FSESSION || prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) && !bpf_jit_supports_fsession()) { bpf_log(log, "JIT does not support fsession\n"); return -EOPNOTSUPP; } if (!btf_type_is_func(t)) { bpf_log(log, "attach_btf_id %u is not a function\n", btf_id); return -EINVAL; } if (prog_extension && btf_check_type_match(log, prog, btf, t)) return -EINVAL; t = btf_attach_func_proto(log, btf, btf_id); if (!t || !btf_type_is_func_proto(t)) return -EINVAL;
if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) return -EINVAL;
if (tgt_prog && conservative) t = NULL;
ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) return ret;
/* * *.multi programs don't need an address during program * verification, we just take the module ref if needed. */ if (is_tracing_multi_id(prog, btf_id)) { if (btf_is_module(btf)) { mod = btf_try_get_module(btf); if (!mod) return -ENOENT; } addr = 0; } else if (tgt_prog) { if (subprog == 0) addr = (long) tgt_prog->bpf_func; else addr = (long) tgt_prog->aux->func[subprog]->bpf_func; } else { if (btf_is_module(btf)) { mod = btf_try_get_module(btf); if (mod) addr = find_kallsyms_symbol_value(mod, tname); else addr = 0; } else { addr = kallsyms_lookup_name(tname); } if (!addr) { module_put(mod); bpf_log(log, "The address of function %s cannot be found\n", tname); return -ENOENT; } }
if (prog->sleepable) { ret = btf_id_allow_sleepable(btf_id, addr, prog, btf); if (ret) { module_put(mod); bpf_log(log, "%s is not sleepable\n", tname); return ret; } } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { if (tgt_prog) { module_put(mod); bpf_log(log, "can't modify return codes of BPF programs\n"); return -EINVAL; } ret = -EINVAL; if (btf_kfunc_is_modify_return(btf, btf_id, prog) || !check_attach_modify_return(addr, tname)) ret = 0; if (ret) { module_put(mod); bpf_log(log, "%s() is not modifiable\n", tname); return ret; } }
break; } tgt_info->tgt_addr = addr; tgt_info->tgt_name = tname; tgt_info->tgt_type = t; tgt_info->tgt_mod = mod; return 0;}
BTF_SET_START(btf_id_deny)BTF_ID_UNUSED#ifdef CONFIG_SMPBTF_ID(func, ___migrate_enable)BTF_ID(func, migrate_disable)BTF_ID(func, migrate_enable)#endif#if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCUBTF_ID(func, rcu_read_unlock_strict)#endif#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)BTF_ID(func, preempt_count_add)BTF_ID(func, preempt_count_sub)#endif#ifdef CONFIG_PREEMPT_RCUBTF_ID(func, __rcu_read_lock)BTF_ID(func, __rcu_read_unlock)#endifBTF_SET_END(btf_id_deny)
/* fexit and fmod_ret can't be used to attach to __noreturn functions. * Currently, we must manually list all __noreturn functions here. Once a more * robust solution is implemented, this workaround can be removed. */BTF_SET_START(noreturn_deny)#ifdef CONFIG_IA32_EMULATIONBTF_ID(func, __ia32_sys_exit)BTF_ID(func, __ia32_sys_exit_group)#endif#ifdef CONFIG_KUNITBTF_ID(func, __kunit_abort)BTF_ID(func, kunit_try_catch_throw)#endif#ifdef CONFIG_MODULESBTF_ID(func, __module_put_and_kthread_exit)#endif#ifdef CONFIG_X86_64BTF_ID(func, __x64_sys_exit)BTF_ID(func, __x64_sys_exit_group)#endifBTF_ID(func, do_exit)BTF_ID(func, do_group_exit)BTF_ID(func, kthread_complete_and_exit)BTF_ID(func, make_task_dead)BTF_SET_END(noreturn_deny)
static bool can_be_sleepable(struct bpf_prog *prog){ if (prog->type == BPF_PROG_TYPE_TRACING) { switch (prog->expected_attach_type) { case BPF_TRACE_FENTRY: case BPF_TRACE_FEXIT: case BPF_MODIFY_RETURN: case BPF_TRACE_ITER: case BPF_TRACE_FSESSION: case BPF_TRACE_RAW_TP: case BPF_TRACE_FENTRY_MULTI: case BPF_TRACE_FEXIT_MULTI: case BPF_TRACE_FSESSION_MULTI: return true; default: return false; } } if (prog->type == BPF_PROG_TYPE_LSM) return prog->expected_attach_type != BPF_LSM_CGROUP;
return prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || prog->type == BPF_PROG_TYPE_STRUCT_OPS || prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || prog->type == BPF_PROG_TYPE_TRACEPOINT;}
static int check_attach_btf_id(struct bpf_verifier_env *env){ struct bpf_prog *prog = env->prog; struct bpf_prog *tgt_prog = prog->aux->dst_prog; struct bpf_attach_target_info tgt_info = {}; u32 btf_id = prog->aux->attach_btf_id; struct bpf_trampoline *tr; int ret; u64 key;
if (prog->type == BPF_PROG_TYPE_SYSCALL) { if (prog->sleepable) /* attach_btf_id checked to be zero already */ return 0; verbose(env, "Syscall programs can only be sleepable\n"); return -EINVAL; }
if (prog->sleepable && !can_be_sleepable(prog)) { verbose(env, "Program of this type cannot be sleepable\n"); return -EINVAL; }
if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) return check_struct_ops_btf_id(env);
if (prog->type != BPF_PROG_TYPE_TRACING && prog->type != BPF_PROG_TYPE_LSM && prog->type != BPF_PROG_TYPE_EXT) return 0;
ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); if (ret) return ret;
if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { /* to make freplace equivalent to their targets, they need to * inherit env->ops and expected_attach_type for the rest of the * verification */ env->ops = bpf_verifier_ops[tgt_prog->type]; prog->expected_attach_type = tgt_prog->expected_attach_type; }
/* store info about the attachment target that will be used later */ prog->aux->attach_func_proto = tgt_info.tgt_type; prog->aux->attach_func_name = tgt_info.tgt_name; prog->aux->mod = tgt_info.tgt_mod;
if (tgt_prog) { prog->aux->saved_dst_prog_type = tgt_prog->type; prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; }
if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { prog->aux->attach_btf_trace = true; return 0; } else if (prog->expected_attach_type == BPF_TRACE_ITER) { return bpf_iter_prog_supported(prog); }
if (prog->type == BPF_PROG_TYPE_LSM) { ret = bpf_lsm_verify_prog(&env->log, prog); if (ret < 0) return ret; } else if (prog->type == BPF_PROG_TYPE_TRACING && btf_id_set_contains(&btf_id_deny, btf_id)) { verbose(env, "Attaching tracing programs to function '%s' is rejected.\n", tgt_info.tgt_name); return -EINVAL; } else if ((prog->expected_attach_type == BPF_TRACE_FEXIT || prog->expected_attach_type == BPF_TRACE_FSESSION || prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI || prog->expected_attach_type == BPF_MODIFY_RETURN) && btf_id_set_contains(&noreturn_deny, btf_id)) { verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n", tgt_info.tgt_name); return -EINVAL; }
/* * We don't get trampoline for tracing_multi programs at this point, * it's done when tracing_multi link is created. */ if (prog->type == BPF_PROG_TYPE_TRACING && is_tracing_multi(prog->expected_attach_type)) return 0;
key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); tr = bpf_trampoline_get(key, &tgt_info); if (!tr) return -ENOMEM;
if (tgt_prog && tgt_prog->aux->tail_call_reachable) tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX;
prog->aux->dst_trampoline = tr; return 0;}
int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id, struct bpf_attach_target_info *tgt_info){ const struct btf_type *t; unsigned long addr; const char *tname; int err;
if (!btf_id || !btf) return -EINVAL;
/* Check noreturn attachment. */ if ((prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) && btf_id_set_contains(&noreturn_deny, btf_id)) return -EINVAL; /* Check denied attachment. */ if (btf_id_set_contains(&btf_id_deny, btf_id)) return -EINVAL;
/* Check and get function target data. */ t = btf_type_by_id(btf, btf_id); if (!t) return -EINVAL; tname = btf_name_by_offset(btf, t->name_off); if (!tname) return -EINVAL; t = btf_attach_func_proto(NULL, btf, btf_id); if (!t || !btf_type_is_func_proto(t)) return -EINVAL; err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); if (err < 0) return err; if (btf_is_module(btf)) { /* The bpf program already holds reference to module. */ if (WARN_ON_ONCE(!prog->aux->mod)) return -EINVAL; addr = find_kallsyms_symbol_value(prog->aux->mod, tname); } else { addr = kallsyms_lookup_name(tname); } if (!addr || !ftrace_location(addr)) return -ENOENT;
/* Check sleepable program attachment. */ if (prog->sleepable) { err = btf_id_allow_sleepable(btf_id, addr, prog, btf); if (err) return err; } tgt_info->tgt_addr = addr; return 0;}
struct btf *bpf_get_btf_vmlinux(void){ if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { mutex_lock(&bpf_verifier_lock); if (!btf_vmlinux) btf_vmlinux = btf_parse_vmlinux(); mutex_unlock(&bpf_verifier_lock); } return btf_vmlinux;}
/* * The add_fd_from_fd_array() is executed only if fd_array_cnt is non-zero. In * this case expect that every file descriptor in the array is either a map or * a BTF. Everything else is considered to be trash. */static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd){ struct bpf_map *map; struct btf *btf; CLASS(fd, f)(fd); int err;
map = __bpf_map_get(f); if (!IS_ERR(map)) { err = __add_used_map(env, map); if (err < 0) return err; return 0; }
btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { btf_get(btf); return __add_used_btf(env, btf); }
verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); return PTR_ERR(map);}
static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr){ size_t size = sizeof(int); int ret; int fd; u32 i;
env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
/* * The only difference between old (no fd_array_cnt is given) and new * APIs is that in the latter case the fd_array is expected to be * continuous and is scanned for map fds right away */ if (!attr->fd_array_cnt) return 0;
/* Check for integer overflow */ if (attr->fd_array_cnt >= (U32_MAX / size)) { verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); return -EINVAL; }
for (i = 0; i < attr->fd_array_cnt; i++) { if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) return -EFAULT;
ret = add_fd_from_fd_array(env, fd); if (ret) return ret; }
return 0;}
/* replace a generic kfunc with a specialized version if necessary */static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx){ struct bpf_prog *prog = env->prog; bool seen_direct_write; void *xdp_kfunc; bool is_rdonly; u32 func_id = desc->func_id; u16 offset = desc->offset; unsigned long addr = desc->addr;
if (offset) /* return if module BTF is used */ return 0;
if (bpf_dev_bound_kfunc_id(func_id)) { xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id); if (xdp_kfunc) addr = (unsigned long)xdp_kfunc; /* fallback to default kfunc when not supported by netdev */ } else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { seen_direct_write = env->seen_direct_write; is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
if (is_rdonly) addr = (unsigned long)bpf_dynptr_from_skb_rdonly;
/* restore env->seen_direct_write to its original value, since * may_access_direct_pkt_data mutates it */ env->seen_direct_write = seen_direct_write; } else if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr]) { if (bpf_lsm_has_d_inode_locked(prog)) addr = (unsigned long)bpf_set_dentry_xattr_locked; } else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) { if (bpf_lsm_has_d_inode_locked(prog)) addr = (unsigned long)bpf_remove_dentry_xattr_locked; } else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { if (!env->insn_aux_data[insn_idx].non_sleepable) addr = (unsigned long)bpf_dynptr_from_file_sleepable; } else if (func_id == special_kfunc_list[KF_bpf_arena_alloc_pages]) { if (env->insn_aux_data[insn_idx].non_sleepable) addr = (unsigned long)bpf_arena_alloc_pages_non_sleepable; } else if (func_id == special_kfunc_list[KF_bpf_arena_free_pages]) { if (env->insn_aux_data[insn_idx].non_sleepable) addr = (unsigned long)bpf_arena_free_pages_non_sleepable; } desc->addr = addr; return 0;}
static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, struct bpf_insn *insn, struct bpf_insn *insn_buf, int *cnt){ struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta; struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) };
insn_buf[0] = addr[0]; insn_buf[1] = addr[1]; insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off); insn_buf[3] = *insn; *cnt = 4;}
int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt){ struct bpf_kfunc_desc *desc; int err;
if (!insn->imm) { verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); return -EINVAL; }
*cnt = 0;
/* insn->imm has the btf func_id. Replace it with an offset relative to * __bpf_call_base, unless the JIT needs to call functions that are * further than 32 bits away (bpf_jit_supports_far_kfunc_call()). */ desc = find_kfunc_desc(env->prog, insn->imm, insn->off); if (!desc) { verifier_bug(env, "kernel function descriptor not found for func_id %u", insn->imm); return -EFAULT; }
err = specialize_kfunc(env, desc, insn_idx); if (err) return err;
if (!bpf_jit_supports_far_kfunc_call()) insn->imm = BPF_CALL_IMM(desc->addr);
if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;
if (is_bpf_percpu_obj_new_kfunc(desc->func_id) && kptr_struct_meta) { verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d", insn_idx); return -EFAULT; }
insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); insn_buf[1] = addr[0]; insn_buf[2] = addr[1]; insn_buf[3] = *insn; *cnt = 4; } else if (is_bpf_obj_drop_kfunc(desc->func_id) || is_bpf_percpu_obj_drop_kfunc(desc->func_id) || is_bpf_refcount_acquire_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
if (is_bpf_percpu_obj_drop_kfunc(desc->func_id) && kptr_struct_meta) { verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d", insn_idx); return -EFAULT; }
if (is_bpf_refcount_acquire_kfunc(desc->func_id) && !kptr_struct_meta) { verifier_bug(env, "kptr_struct_meta expected at insn_idx %d", insn_idx); return -EFAULT; }
insn_buf[0] = addr[0]; insn_buf[1] = addr[1]; insn_buf[2] = *insn; *cnt = 3; } else if (is_bpf_list_push_kfunc(desc->func_id) || is_bpf_rbtree_add_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; int struct_meta_reg = BPF_REG_3; int node_offset_reg = BPF_REG_4;
/* list_add/rbtree_add have an extra arg (prev/less), * so args-to-fixup are in diff regs. */ if (desc->func_id == special_kfunc_list[KF_bpf_list_add] || is_bpf_rbtree_add_kfunc(desc->func_id)) { struct_meta_reg = BPF_REG_4; node_offset_reg = BPF_REG_5; }
if (!kptr_struct_meta) { verifier_bug(env, "kptr_struct_meta expected at insn_idx %d", insn_idx); return -EFAULT; }
__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, node_offset_reg, insn, insn_buf, cnt); } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); *cnt = 1; } else if (desc->func_id == special_kfunc_list[KF_bpf_session_is_return] && (env->prog->expected_attach_type == BPF_TRACE_FSESSION || env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) {
/* * inline the bpf_session_is_return() for fsession: * bool bpf_session_is_return(void *ctx) * { * return (((u64 *)ctx)[-1] >> BPF_TRAMP_IS_RETURN_SHIFT) & 1; * } */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_IS_RETURN_SHIFT); insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1); *cnt = 3; } else if (desc->func_id == special_kfunc_list[KF_bpf_session_cookie] && (env->prog->expected_attach_type == BPF_TRACE_FSESSION || env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { /* * inline bpf_session_cookie() for fsession: * __u64 *bpf_session_cookie(void *ctx) * { * u64 off = (((u64 *)ctx)[-1] >> BPF_TRAMP_COOKIE_INDEX_SHIFT) & 0xFF; * return &((u64 *)ctx)[-off]; * } */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_COOKIE_INDEX_SHIFT); insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); insn_buf[3] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1); insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0); *cnt = 6; }
if (env->insn_aux_data[insn_idx].arg_prog) { u32 regno = env->insn_aux_data[insn_idx].arg_prog; struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) }; int idx = *cnt;
insn_buf[idx++] = ld_addrs[0]; insn_buf[idx++] = ld_addrs[1]; insn_buf[idx++] = *insn; *cnt = idx; } return 0;}
int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log){ u64 start_time = ktime_get_ns(); struct bpf_verifier_env *env; int i, len, ret = -EINVAL, err; bool is_priv;
BTF_TYPE_EMIT(enum bpf_features);
/* no program is valid */ if (ARRAY_SIZE(bpf_verifier_ops) == 0) return -EINVAL;
/* 'struct bpf_verifier_env' can be global, but since it's not small, * allocate/free it every time bpf_check() is called */ env = kvzalloc_obj(struct bpf_verifier_env, GFP_KERNEL_ACCOUNT); if (!env) return -ENOMEM;
env->bt.env = env;
len = (*prog)->len; env->insn_aux_data = vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); ret = -ENOMEM; if (!env->insn_aux_data) goto err_free_env; for (i = 0; i < len; i++) env->insn_aux_data[i].orig_idx = i; env->succ = bpf_iarray_realloc(NULL, 2); if (!env->succ) goto err_free_env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type];
env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
bpf_get_btf_vmlinux();
/* grab the mutex to protect few globals used by verifier */ if (!is_priv) mutex_lock(&bpf_verifier_lock);
/* user could have requested verbose verifier output * and supplied buffer to store the verification trace */ ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) goto err_unlock;
ret = process_fd_array(env, attr, uattr); if (ret) goto skip_full_check;
mark_verifier_state_clean(env);
if (IS_ERR(btf_vmlinux)) { /* Either gcc or pahole or kernel are broken. */ verbose(env, "in-kernel BTF is malformed\n"); ret = PTR_ERR(btf_vmlinux); goto skip_full_check; }
env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) env->strict_alignment = true; if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) env->strict_alignment = false;
if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), GFP_KERNEL_ACCOUNT); ret = -ENOMEM; if (!env->explored_states) goto skip_full_check;
for (i = 0; i < state_htab_size(env); i++) INIT_LIST_HEAD(&env->explored_states[i]); INIT_LIST_HEAD(&env->free_list);
ret = bpf_check_btf_info_early(env, attr, uattr); if (ret < 0) goto skip_full_check;
ret = add_subprog_and_kfunc(env); if (ret < 0) goto skip_full_check;
ret = check_subprogs(env); if (ret < 0) goto skip_full_check;
ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check;
ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check;
if (bpf_prog_is_offloaded(env->prog->aux)) { ret = bpf_prog_offload_verifier_prep(env->prog); if (ret) goto skip_full_check; }
ret = bpf_check_cfg(env); if (ret < 0) goto skip_full_check;
ret = bpf_compute_postorder(env); if (ret < 0) goto skip_full_check;
ret = bpf_stack_liveness_init(env); if (ret) goto skip_full_check;
ret = check_attach_btf_id(env); if (ret) goto skip_full_check;
ret = bpf_compute_const_regs(env); if (ret < 0) goto skip_full_check;
ret = bpf_prune_dead_branches(env); if (ret < 0) goto skip_full_check;
ret = sort_subprogs_topo(env); if (ret < 0) goto skip_full_check;
ret = bpf_compute_scc(env); if (ret < 0) goto skip_full_check;
ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check;
ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check;
ret = do_check_main(env); ret = ret ?: do_check_subprogs(env);
if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env);
skip_full_check: kvfree(env->explored_states);
/* might decrease stack depth, keep it before passes that * allocate additional slots. */ if (ret == 0) ret = bpf_remove_fastcall_spills_fills(env);
if (ret == 0) ret = check_max_stack_depth(env);
/* instruction rewrites happen after this point */ if (ret == 0) ret = bpf_optimize_bpf_loop(env);
if (is_priv) { if (ret == 0) bpf_opt_hard_wire_dead_code_branches(env); if (ret == 0) ret = bpf_opt_remove_dead_code(env); if (ret == 0) ret = bpf_opt_remove_nops(env); } else { if (ret == 0) sanitize_dead_code(env); }
if (ret == 0) /* program is valid, convert *(u32*)(ctx + off) accesses */ ret = bpf_convert_ctx_accesses(env);
if (ret == 0) ret = bpf_do_misc_fixups(env);
/* do 32-bit optimization after insn patching has done so those patched * insns could be handled correctly. */ if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { ret = bpf_opt_subreg_zext_lo32_rnd_hi32(env, attr); env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret : false; }
if (ret == 0) ret = bpf_fixup_call_args(env);
env->verification_time = ktime_get_ns() - start_time; print_verification_stats(env); env->prog->aux->verified_insns = env->insn_processed;
/* preserve original error even if log finalization is successful */ err = bpf_log_attr_finalize(attr_log, &env->log); if (err) ret = err;
if (ret) goto err_release_maps;
if (env->used_map_cnt) { /* if program passed verifier, update used_maps in bpf_prog_info */ env->prog->aux->used_maps = kmalloc_objs(env->used_maps[0], env->used_map_cnt, GFP_KERNEL_ACCOUNT);
if (!env->prog->aux->used_maps) { ret = -ENOMEM; goto err_release_maps; }
memcpy(env->prog->aux->used_maps, env->used_maps, sizeof(env->used_maps[0]) * env->used_map_cnt); env->prog->aux->used_map_cnt = env->used_map_cnt; } if (env->used_btf_cnt) { /* if program passed verifier, update used_btfs in bpf_prog_aux */ env->prog->aux->used_btfs = kmalloc_objs(env->used_btfs[0], env->used_btf_cnt, GFP_KERNEL_ACCOUNT); if (!env->prog->aux->used_btfs) { ret = -ENOMEM; goto err_release_maps; }
memcpy(env->prog->aux->used_btfs, env->used_btfs, sizeof(env->used_btfs[0]) * env->used_btf_cnt); env->prog->aux->used_btf_cnt = env->used_btf_cnt; } if (env->used_map_cnt || env->used_btf_cnt) { /* program is valid. Convert pseudo bpf_ld_imm64 into generic * bpf_ld_imm64 instructions */ convert_pseudo_ld_imm64(env); }
adjust_btf_func(env);
/* extension progs temporarily inherit the attach_type of their targets for verification purposes, so set it back to zero before returning */ if (env->prog->type == BPF_PROG_TYPE_EXT) env->prog->expected_attach_type = 0;
env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
err_release_maps: if (ret) release_insn_arrays(env); if (!env->prog->aux->used_maps) /* if we didn't copy map pointers into bpf_prog_info, release * them now. Otherwise free_used_maps() will release them. */ release_maps(env); if (!env->prog->aux->used_btfs) release_btfs(env);
*prog = env->prog;
module_put(env->attach_btf_mod);err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len);err_free_env: bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); vfree(env->insn_aux_data); kvfree(env); return ret;}