Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794// SPDX-License-Identifier: GPL-2.0-only/* * linux/mm/memory.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */
/* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */
/* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */
/* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. * Found it. Everything seems to work now. * 20.12.91 - Ok, making the swap-device changeable like the root. */
/* * 05.04.94 - Multi-page memory management added for v1.1. * Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG * (Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */
#include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/mm_inline.h>#include <linux/sched/mm.h>#include <linux/sched/numa_balancing.h>#include <linux/sched/task.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/memremap.h>#include <linux/kmsan.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/export.h>#include <linux/delayacct.h>#include <linux/init.h>#include <linux/writeback.h>#include <linux/memcontrol.h>#include <linux/mmu_notifier.h>#include <linux/leafops.h>#include <linux/elf.h>#include <linux/gfp.h>#include <linux/migrate.h>#include <linux/string.h>#include <linux/shmem_fs.h>#include <linux/memory-tiers.h>#include <linux/debugfs.h>#include <linux/userfaultfd_k.h>#include <linux/dax.h>#include <linux/oom.h>#include <linux/numa.h>#include <linux/perf_event.h>#include <linux/ptrace.h>#include <linux/vmalloc.h>#include <linux/sched/sysctl.h>#include <linux/pgalloc.h>#include <linux/uaccess.h>
#include <trace/events/kmem.h>
#include <asm/io.h>#include <asm/mmu_context.h>#include <asm/tlb.h>#include <asm/tlbflush.h>
#include "pgalloc-track.h"#include "internal.h"#include "swap.h"
#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.#endif
static vm_fault_t do_fault(struct vm_fault *vmf);static vm_fault_t do_anonymous_page(struct vm_fault *vmf);static bool vmf_pte_changed(struct vm_fault *vmf);
/* * Return true if the original pte was a uffd-wp pte marker (so the pte was * wr-protected). */static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf){ if (!userfaultfd_wp(vmf->vma)) return false; if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)) return false;
return pte_is_uffd_wp_marker(vmf->orig_pte);}
/* * Randomize the address space (stacks, mmaps, brk, etc.). * * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, * as ancient (libc5 based) binaries can segfault. ) */int randomize_va_space __read_mostly =#ifdef CONFIG_COMPAT_BRK 1;#else 2;#endif
static const struct ctl_table mmu_sysctl_table[] = { { .procname = "randomize_va_space", .data = &randomize_va_space, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, },};
static int __init init_mm_sysctl(void){ register_sysctl_init("kernel", mmu_sysctl_table); return 0;}
subsys_initcall(init_mm_sysctl);
#ifndef arch_wants_old_prefaulted_ptestatic inline bool arch_wants_old_prefaulted_pte(void){ /* * Transitioning a PTE from 'old' to 'young' can be expensive on * some architectures, even if it's performed in hardware. By * default, "false" means prefaulted entries will be 'young'. */ return false;}#endif
static int __init disable_randmaps(char *s){ randomize_va_space = 0; return 1;}__setup("norandmaps", disable_randmaps);
unsigned long highest_memmap_pfn __read_mostly;
void mm_trace_rss_stat(struct mm_struct *mm, int member){ trace_rss_stat(mm, member);}
/* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, unsigned long addr){ pgtable_t token = pmd_pgtable(*pmd); pmd_clear(pmd); pte_free_tlb(tlb, token, addr); mm_dec_nr_ptes(tlb->mm);}
static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling){ pmd_t *pmd; unsigned long next; unsigned long start;
start = addr; pmd = pmd_offset(pud, addr); do { next = pmd_addr_end(addr, end); if (pmd_none_or_clear_bad(pmd)) continue; free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end);
start &= PUD_MASK; if (start < floor) return; if (ceiling) { ceiling &= PUD_MASK; if (!ceiling) return; } if (end - 1 > ceiling - 1) return;
pmd = pmd_offset(pud, start); pud_clear(pud); pmd_free_tlb(tlb, pmd, start); mm_dec_nr_pmds(tlb->mm);}
static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling){ pud_t *pud; unsigned long next; unsigned long start;
start = addr; pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) continue; free_pmd_range(tlb, pud, addr, next, floor, ceiling); } while (pud++, addr = next, addr != end);
start &= P4D_MASK; if (start < floor) return; if (ceiling) { ceiling &= P4D_MASK; if (!ceiling) return; } if (end - 1 > ceiling - 1) return;
pud = pud_offset(p4d, start); p4d_clear(p4d); pud_free_tlb(tlb, pud, start); mm_dec_nr_puds(tlb->mm);}
static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling){ p4d_t *p4d; unsigned long next; unsigned long start;
start = addr; p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(p4d)) continue; free_pud_range(tlb, p4d, addr, next, floor, ceiling); } while (p4d++, addr = next, addr != end);
start &= PGDIR_MASK; if (start < floor) return; if (ceiling) { ceiling &= PGDIR_MASK; if (!ceiling) return; } if (end - 1 > ceiling - 1) return;
p4d = p4d_offset(pgd, start); pgd_clear(pgd); p4d_free_tlb(tlb, p4d, start);}
/** * free_pgd_range - Unmap and free page tables in the range * @tlb: the mmu_gather containing pending TLB flush info * @addr: virtual address start * @end: virtual address end * @floor: lowest address boundary * @ceiling: highest address boundary * * This function tears down all user-level page tables in the * specified virtual address range [@addr..@end). It is part of * the memory unmap flow. */void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling){ pgd_t *pgd; unsigned long next;
/* * The next few lines have given us lots of grief... * * Why are we testing PMD* at this top level? Because often * there will be no work to do at all, and we'd prefer not to * go all the way down to the bottom just to discover that. * * Why all these "- 1"s? Because 0 represents both the bottom * of the address space and the top of it (using -1 for the * top wouldn't help much: the masks would do the wrong thing). * The rule is that addr 0 and floor 0 refer to the bottom of * the address space, but end 0 and ceiling 0 refer to the top * Comparisons need to use "end - 1" and "ceiling - 1" (though * that end 0 case should be mythical). * * Wherever addr is brought up or ceiling brought down, we must * be careful to reject "the opposite 0" before it confuses the * subsequent tests. But what about where end is brought down * by PMD_SIZE below? no, end can't go down to 0 there. * * Whereas we round start (addr) and ceiling down, by different * masks at different levels, in order to test whether a table * now has no other vmas using it, so can be freed, we don't * bother to round floor or end up - the tests don't need that. */
addr &= PMD_MASK; if (addr < floor) { addr += PMD_SIZE; if (!addr) return; } if (ceiling) { ceiling &= PMD_MASK; if (!ceiling) return; } if (end - 1 > ceiling - 1) end -= PMD_SIZE; if (addr > end - 1) return; /* * We add page table cache pages with PAGE_SIZE, * (see pte_free_tlb()), flush the tlb if we need */ tlb_change_page_size(tlb, PAGE_SIZE); pgd = pgd_offset(tlb->mm, addr); do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; free_p4d_range(tlb, pgd, addr, next, floor, ceiling); } while (pgd++, addr = next, addr != end);}
/** * free_pgtables() - Free a range of page tables * @tlb: The mmu gather * @unmap: The unmap_desc * * Note: pg_start and pg_end are provided to indicate the absolute range of the * page tables that should be removed. This can differ from the vma mappings on * some archs that may have mappings that need to be removed outside the vmas. * Note that the prev->vm_end and next->vm_start are often used. * * The vma_end differs from the pg_end when a dup_mmap() failed and the tree has * unrelated data to the mm_struct being torn down. */void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *unmap){ struct unlink_vma_file_batch vb; struct ma_state *mas = unmap->mas; struct vm_area_struct *vma = unmap->first;
/* * Note: USER_PGTABLES_CEILING may be passed as the value of pg_end and * may be 0. Underflow is expected in this case. Otherwise the * pagetable end is exclusive. vma_end is exclusive. The last vma * address should never be larger than the pagetable end. */ WARN_ON_ONCE(unmap->vma_end - 1 > unmap->pg_end - 1);
tlb_free_vmas(tlb);
do { unsigned long addr = vma->vm_start; struct vm_area_struct *next;
next = mas_find(mas, unmap->tree_end - 1);
/* * Hide vma from rmap and truncate_pagecache before freeing * pgtables */ if (unmap->mm_wr_locked) vma_start_write(vma); unlink_anon_vmas(vma);
unlink_file_vma_batch_init(&vb); unlink_file_vma_batch_add(&vb, vma);
/* * Optimization: gather nearby vmas into one call down */ while (next && next->vm_start <= vma->vm_end + PMD_SIZE) { vma = next; next = mas_find(mas, unmap->tree_end - 1); if (unmap->mm_wr_locked) vma_start_write(vma); unlink_anon_vmas(vma); unlink_file_vma_batch_add(&vb, vma); } unlink_file_vma_batch_final(&vb);
free_pgd_range(tlb, addr, vma->vm_end, unmap->pg_start, next ? next->vm_start : unmap->pg_end); vma = next; } while (vma);}
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte){ spinlock_t *ptl = pmd_lock(mm, pmd);
if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ mm_inc_nr_ptes(mm); /* * Ensure all pte setup (eg. pte page lock and page clearing) are * visible before the pte is made visible to other CPUs by being * put into page tables. * * The other side of the story is the pointer chasing in the page * table walking code (when walking the page table without locking; * ie. most of the time). Fortunately, these data accesses consist * of a chain of data-dependent loads, meaning most CPUs (alpha * being the notable exception) will already guarantee loads are * seen in-order. See the alpha page table accessors for the * smp_rmb() barriers in page table walking code. */ smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ pmd_populate(mm, pmd, *pte); *pte = NULL; } spin_unlock(ptl);}
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd){ pgtable_t new = pte_alloc_one(mm); if (!new) return -ENOMEM;
pmd_install(mm, pmd, &new); if (new) pte_free(mm, new); return 0;}
int __pte_alloc_kernel(pmd_t *pmd){ pte_t *new = pte_alloc_one_kernel(&init_mm); if (!new) return -ENOMEM;
spin_lock(&init_mm.page_table_lock); if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ smp_wmb(); /* See comment in pmd_install() */ pmd_populate_kernel(&init_mm, pmd, new); new = NULL; } spin_unlock(&init_mm.page_table_lock); if (new) pte_free_kernel(&init_mm, new); return 0;}
static inline void init_rss_vec(int *rss){ memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);}
static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss){ int i;
for (i = 0; i < NR_MM_COUNTERS; i++) if (rss[i]) add_mm_counter(mm, i, rss[i]);}
/* Allow a burst of 60 bad page map reports per minute. */static DEFINE_RATELIMIT_STATE(bad_page_map_ratelimit, 60 * HZ, 60);
void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size){ if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) { snprintf(buf, buf_size, "overflow"); return; }
switch (entry_size) { case sizeof(u32): snprintf(buf, buf_size, "%08x", *(const u32 *)entry); break; case sizeof(u64): snprintf(buf, buf_size, "%016llx", *(const u64 *)entry); break;#if defined(__SIZEOF_INT128__) case sizeof(u128): snprintf(buf, buf_size, "%016llx%016llx", (unsigned long long)(*(const u128 *)entry >> 64), (unsigned long long)*(const u128 *)entry); break;#endif default: snprintf(buf, buf_size, "unsupported"); break; }}
static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr){ char pgd_str[PTVAL_STR_MAX]; char p4d_str[PTVAL_STR_MAX]; char pud_str[PTVAL_STR_MAX]; char pmd_str[PTVAL_STR_MAX]; p4d_t p4d, *p4dp; pud_t pud, *pudp; pmd_t pmd, *pmdp; pgd_t *pgdp;
/* * Although this looks like a fully lockless pgtable walk, it is not: * see locking requirements for print_bad_page_map(). */ pgdp = pgd_offset(mm, addr); ptval_to_str(pgd_str, pgd_val(*pgdp));
if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) { pr_alert("pgd:%s\n", pgd_str); return; }
p4dp = p4d_offset(pgdp, addr); p4d = p4dp_get(p4dp); ptval_to_str(p4d_str, p4d_val(p4d));
if (!p4d_present(p4d) || p4d_leaf(p4d)) { pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str); return; }
pudp = pud_offset(p4dp, addr); pud = pudp_get(pudp); ptval_to_str(pud_str, pud_val(pud));
if (!pud_present(pud) || pud_leaf(pud)) { pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str); return; }
pmdp = pmd_offset(pudp, addr); pmd = pmdp_get(pmdp); ptval_to_str(pmd_str, pmd_val(pmd));
/* * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE, * because the table should already be mapped by the caller and * doing another map would be bad. print_bad_page_map() should * already take care of printing the PTE. */ pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str);}
/* * This function is called to print an error when a bad page table entry (e.g., * corrupted page table entry) is found. For example, we might have a * PFN-mapped pte in a region that doesn't allow it. * * The calling function must still handle the error. * * This function must be called during a proper page table walk, as it will * re-walk the page table to dump information: the caller MUST prevent page * table teardown (by holding mmap, vma or rmap lock) and MUST hold the leaf * page table lock. */static void print_bad_page_map(struct vm_area_struct *vma, unsigned long addr, const void *entry, size_t entry_size, struct page *page, enum pgtable_level level){ struct address_space *mapping; char entry_str[PTVAL_STR_MAX]; pgoff_t index, anon_index;
if (!__ratelimit(&bad_page_map_ratelimit)) return;
mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; index = linear_page_index(vma, addr); anon_index = __linear_anon_page_index(vma, addr);
ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size); pr_alert("BUG: Bad page map in process %s %s:%s", current->comm, pgtable_level_to_str(level), entry_str); __print_bad_page_map_pgtable(vma->vm_mm, addr); if (page) dump_page(page, "bad page map"); pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px", (void *)addr, vma->vm_flags, vma->anon_vma, mapping); if (!vma_is_cow_mapping(vma) || index == anon_index) { pr_cont(" index:%lx\n", index); } else { pr_cont(" index:%lx (file) %lx (anon)\n", index, anon_index); }
pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n", vma->vm_file, vma->vm_ops ? vma->vm_ops->fault : NULL, vma->vm_file ? vma->vm_file->f_op->mmap : NULL, vma->vm_file ? vma->vm_file->f_op->mmap_prepare : NULL, mapping ? mapping->a_ops->read_folio : NULL); dump_stack(); add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);}
static inline bool pgtable_level_has_pxx_special(enum pgtable_level level){ switch (level) { case PGTABLE_LEVEL_PTE: return IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL); case PGTABLE_LEVEL_PMD: return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP); case PGTABLE_LEVEL_PUD: return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP); default: return false; }}
static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, pte_t pte, struct page *page){ auto entry = pte_val(pte);
print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE);}
/** * __vm_normal_page() - Get the "struct page" associated with a page table entry. * @vma: The VMA mapping the page table entry. * @addr: The address where the page table entry is mapped. * @pfn: The PFN stored in the page table entry. * @special: Whether the page table entry is marked "special". * @entry: The page table entry value for error reporting purposes only. * @entry_size: The size of @entry. * @level: The page table level for error reporting purposes only. * * "Special" mappings do not wish to be associated with a "struct page" (either * it doesn't exist, or it exists but they don't want to touch it). In this * case, NULL is returned here. "Normal" mappings do have a struct page and * are ordinarily refcounted. * * Page mappings of the shared zero folios are always considered "special", as * they are not ordinarily refcounted: neither the refcount nor the mapcount * of these folios is adjusted when mapping them into user page tables. * Selected page table walkers (such as GUP) can still identify mappings of the * shared zero folios and work with the underlying "struct page". * * There are 2 broad cases. Firstly, an architecture may define a "special" * page table entry bit, such as pte_special(), in which case this function is * trivial. Secondly, an architecture may not have a spare page table * entry bit, which requires a more complicated scheme, described below. * * With CONFIG_FIND_NORMAL_PAGE, we might have the "special" bit set on * page table entries that actually map "normal" pages: however, that page * cannot be looked up through the PFN stored in the page table entry, but * instead will be looked up through vm_ops->find_normal_page(). So far, this * only applies to PTEs. * * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a * special mapping (even if there are underlying and valid "struct pages"). * COWed pages of a VM_PFNMAP are always normal. * * The way we recognize COWed pages within VM_PFNMAP mappings is through the * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit * set, and the vm_pgoff will point to the first PFN mapped: thus every special * mapping will always honor the rule * * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) * * And for normal mappings this is false. * * This restricts such mappings to be a linear translation from virtual address * to pfn. To get around this restriction, we allow arbitrary mappings so long * as the vma is not a COW mapping; in that case, we know that all ptes are * special (because none can have been COWed). * * * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. * * VM_MIXEDMAP mappings can likewise contain memory with or without "struct * page" backing, however the difference is that _all_ pages with a struct * page (that is, those where pfn_valid is true, except the shared zero * folios) are refcounted and considered normal pages by the VM. * * The disadvantage is that pages are refcounted (which can be slower and * simply not an option for some PFNMAP users). The advantage is that we * don't have to follow the strict linearity rule of PFNMAP mappings in * order to support COWable mappings. * * Return: Returns the "struct page" if this is a "normal" mapping. Returns * NULL if this is a "special" mapping. */static inline struct page *__vm_normal_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, bool special, const void *entry, size_t entry_size, enum pgtable_level level){ if (pgtable_level_has_pxx_special(level)) { if (unlikely(special)) {#ifdef CONFIG_FIND_NORMAL_PAGE if (vma->vm_ops && vma->vm_ops->find_normal_page) return vma->vm_ops->find_normal_page(vma, addr);#endif /* CONFIG_FIND_NORMAL_PAGE */ if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) return NULL; if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return NULL;
print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* * With working pte_special()/pmd_special()..., any special page * table mappings (incl. shared zero folios) are marked * accordingly. */ } else { if (unlikely(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))) { if (vma->vm_flags & VM_MIXEDMAP) { /* If it has a "struct page", it's "normal". */ if (!pfn_valid(pfn)) return NULL; } else { const pgoff_t index = linear_page_index(vma, addr);
/* Only CoW'ed anon folios are "normal". */ if (pfn == index) return NULL; if (!vma_is_cow_mapping(vma)) return NULL; } }
if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)) return NULL; }
if (unlikely(pfn > highest_memmap_pfn)) { /* Corrupted page table entry. */ print_bad_page_map(vma, addr, entry, entry_size, NULL, level); return NULL; } /* * NOTE! We still have PageReserved() pages in the page tables. * For example, VDSO mappings can cause them to exist. */ VM_WARN_ON_ONCE(is_zero_pfn(pfn) || is_huge_zero_pfn(pfn)); return pfn_to_page(pfn);}
/** * vm_normal_page() - Get the "struct page" associated with a PTE * @vma: The VMA mapping the @pte. * @addr: The address where the @pte is mapped. * @pte: The PTE. * * Get the "struct page" associated with a PTE. See __vm_normal_page() * for details on "normal" and "special" mappings. * * Return: Returns the "struct page" if this is a "normal" mapping. Returns * NULL if this is a "special" mapping. */struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte){ auto entry = pte_val(pte);
return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte), &entry, sizeof(entry), PGTABLE_LEVEL_PTE);}
/** * vm_normal_folio() - Get the "struct folio" associated with a PTE * @vma: The VMA mapping the @pte. * @addr: The address where the @pte is mapped. * @pte: The PTE. * * Get the "struct folio" associated with a PTE. See __vm_normal_page() * for details on "normal" and "special" mappings. * * Return: Returns the "struct folio" if this is a "normal" mapping. Returns * NULL if this is a "special" mapping. */struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, pte_t pte){ struct page *page = vm_normal_page(vma, addr, pte);
if (page) return page_folio(page); return NULL;}
#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES/** * vm_normal_page_pmd() - Get the "struct page" associated with a PMD * @vma: The VMA mapping the @pmd. * @addr: The address where the @pmd is mapped. * @pmd: The PMD. * * Get the "struct page" associated with a PTE. See __vm_normal_page() * for details on "normal" and "special" mappings. * * Return: Returns the "struct page" if this is a "normal" mapping. Returns * NULL if this is a "special" mapping. */struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd){ auto entry = pmd_val(pmd);
return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd), &entry, sizeof(entry), PGTABLE_LEVEL_PMD);}
/** * vm_normal_folio_pmd() - Get the "struct folio" associated with a PMD * @vma: The VMA mapping the @pmd. * @addr: The address where the @pmd is mapped. * @pmd: The PMD. * * Get the "struct folio" associated with a PTE. See __vm_normal_page() * for details on "normal" and "special" mappings. * * Return: Returns the "struct folio" if this is a "normal" mapping. Returns * NULL if this is a "special" mapping. */struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd){ struct page *page = vm_normal_page_pmd(vma, addr, pmd);
if (page) return page_folio(page); return NULL;}
/** * vm_normal_page_pud() - Get the "struct page" associated with a PUD * @vma: The VMA mapping the @pud. * @addr: The address where the @pud is mapped. * @pud: The PUD. * * Get the "struct page" associated with a PUD. See __vm_normal_page() * for details on "normal" and "special" mappings. * * Return: Returns the "struct page" if this is a "normal" mapping. Returns * NULL if this is a "special" mapping. */struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud){ auto entry = pud_val(pud);
return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud), &entry, sizeof(entry), PGTABLE_LEVEL_PUD);}#endif
/** * restore_exclusive_pte - Restore a device-exclusive entry * @vma: VMA covering @address * @folio: the mapped folio * @page: the mapped folio page * @address: the virtual address * @ptep: pte pointer into the locked page table mapping the folio page * @orig_pte: pte value at @ptep * * Restore a device-exclusive non-swap entry to an ordinary present pte. * * The folio and the page table must be locked, and MMU notifiers must have * been called to invalidate any (exclusive) device mappings. * * Locking the folio makes sure that anybody who just converted the pte to * a device-exclusive entry can map it into the device to make forward * progress without others converting it back until the folio was unlocked. * * If the folio lock ever becomes an issue, we can stop relying on the folio * lock; it might make some scenarios with heavy thrashing less likely to * make forward progress, but these scenarios might not be valid use cases. * * Note that the folio lock does not protect against all cases of concurrent * page table modifications (e.g., MADV_DONTNEED, mprotect), so device drivers * must use MMU notifiers to sync against any concurrent changes. */static void restore_exclusive_pte(struct vm_area_struct *vma, struct folio *folio, struct page *page, unsigned long address, pte_t *ptep, pte_t orig_pte){ pte_t pte;
VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot))); if (pte_swp_soft_dirty(orig_pte)) pte = pte_mksoft_dirty(pte);
if (pte_swp_uffd(orig_pte)) pte = pte_mkuffd(pte);
/* See do_swap_page(): restore PAGE_NONE for RWP */ if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma)) pte = pte_modify(pte, PAGE_NONE);
if ((vma->vm_flags & VM_WRITE) && can_change_pte_writable(vma, address, pte)) { if (folio_test_dirty(folio)) pte = pte_mkdirty(pte); pte = pte_mkwrite(pte, vma); } set_pte_at(vma->vm_mm, address, ptep, pte);
/* * No need to invalidate - it was non-present before. However * secondary CPUs may have mappings that need invalidating. */ update_mmu_cache(vma, address, ptep);}
/* * Tries to restore an exclusive pte if the page lock can be acquired without * sleeping. */static int try_restore_exclusive_pte(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t orig_pte){ const softleaf_t entry = softleaf_from_pte(orig_pte); struct page *page = softleaf_to_page(entry); struct folio *folio = page_folio(page);
if (folio_trylock(folio)) { restore_exclusive_pte(vma, folio, page, addr, ptep, orig_pte); folio_unlock(folio); return 0; }
return -EBUSY;}
/* * copy one vm_area from one task to the other. Assumes the page tables * already present in the new task to be cleared in the whole range * covered by this vma. */
static unsigned longcopy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long addr, int *rss){ pte_t orig_pte = ptep_get(src_pte); softleaf_t entry = softleaf_from_pte(orig_pte); pte_t pte = orig_pte; struct folio *folio; struct page *page;
if (likely(softleaf_is_swap(entry))) { if (swap_dup_entry_direct(entry) < 0) return -EIO;
mm_prepare_for_swap_entries(dst_mm); /* Mark the swap entry as shared. */ if (pte_swp_exclusive(orig_pte)) { pte = pte_swp_clear_exclusive(orig_pte); set_pte_at(src_mm, addr, src_pte, pte); } rss[MM_SWAPENTS]++; } else if (softleaf_is_migration(entry)) { folio = softleaf_to_folio(entry);
rss[mm_counter(folio)]++;
if (!softleaf_is_migration_read(entry) && vma_is_cow_mapping(dst_vma)) { /* * COW mappings require pages in both parent and child * to be set to read. A previously exclusive entry is * now shared. */ entry = make_readable_migration_entry( swp_offset(entry)); pte = softleaf_to_pte(entry); if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); if (pte_swp_uffd(orig_pte)) pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_private(entry)) { page = softleaf_to_page(entry); folio = page_folio(page);
/* * Update rss count even for unaddressable pages, as * they should treated just like normal pages in this * respect. * * We will likely want to have some new rss counters * for unaddressable pages, at some point. But for now * keep things as they are. */ folio_get(folio); rss[mm_counter(folio)]++; /* Cannot fail as these pages cannot get pinned. */ folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma);
/* * We do not preserve soft-dirty information, because so * far, checkpoint/restore is the only feature that * requires that. And checkpoint/restore does not work * when a device driver is involved (you cannot easily * save and restore device driver state). */ if (softleaf_is_device_private_write(entry) && vma_is_cow_mapping(dst_vma)) { entry = make_readable_device_private_entry( swp_offset(entry)); pte = swp_entry_to_pte(entry); if (pte_swp_uffd(orig_pte)) pte = pte_swp_mkuffd(pte); set_pte_at(src_mm, addr, src_pte, pte); } } else if (softleaf_is_device_exclusive(entry)) { /* * Make device exclusive entries present by restoring the * original entry then copying as for a present pte. Device * exclusive entries currently only support private writable * (ie. COW) mappings. */ VM_BUG_ON(!vma_is_cow_mapping(src_vma)); if (try_restore_exclusive_pte(src_vma, addr, src_pte, orig_pte)) return -EBUSY; return -ENOENT; } else if (softleaf_is_marker(entry)) { pte_marker marker = copy_pte_marker(entry, dst_vma);
if (marker) set_pte_at(dst_mm, addr, dst_pte, make_pte_marker(marker)); return 0; } if (!userfaultfd_protected(dst_vma)) pte = pte_swp_clear_uffd(pte); set_pte_at(dst_mm, addr, dst_pte, pte); return 0;}
/* * Copy a present and normal page. * * NOTE! The usual case is that this isn't required; * instead, the caller can just increase the page refcount * and re-use the pte the traditional way. * * And if we need a pre-allocated page but don't yet have * one, return a negative error to let the preallocation * code know so that it can do so outside the page table * lock. */static inline intcopy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, struct folio **prealloc, struct page *page){ struct folio *new_folio; pte_t pte;
new_folio = *prealloc; if (!new_folio) return -EAGAIN;
/* * We have a prealloc page, all good! Take it * over and copy the page & arm it. */
if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma)) return -EHWPOISON;
*prealloc = NULL; __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); rss[MM_ANONPAGES]++;
/* All done, just insert the new page copy in the child */ pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) { /* The uffd bit needs to be delivered to the dest pte as well */ pte = pte_mkuffd(pte); /* Restore PAGE_NONE so the RWP marker keeps trapping */ if (userfaultfd_rwp(dst_vma)) pte = pte_modify(pte, PAGE_NONE); } set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); return 0;}
static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int nr){ struct mm_struct *src_mm = src_vma->vm_mm; bool writable;
/* * Snapshot writability before the RWP-disarm rewrite below: when the * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can * silently drop _PAGE_RW from a resolved (no-marker) writable PTE, * so a later pte_write(pte) check would skip the COW wrprotect and * leave the parent writable over a folio shared with the child. */ writable = pte_write(pte);
/* * Child is not RWP-armed: restore accessible protection so the * inherited PAGE_NONE does not cost a fault on first read. Gate on * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing) * are not normalised away. */ if (!userfaultfd_protected(dst_vma)) { if (userfaultfd_rwp(src_vma) && pte_uffd(pte)) pte = pte_modify(pte, dst_vma->vm_page_prot); pte = pte_clear_uffd(pte); }
/* If it's a COW mapping, write protect it both processes. */ if (vma_is_cow_mapping(src_vma) && writable) { wrprotect_ptes(src_mm, addr, src_pte, nr); pte = pte_wrprotect(pte); }
/* If it's a shared mapping, mark it clean in the child. */ if (src_vma->vm_flags & VM_SHARED) pte = pte_mkclean(pte); pte = pte_mkold(pte);
set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);}
/* * Copy one present PTE, trying to batch-process subsequent PTEs that map * consecutive pages of the same folio by copying them as well. * * Returns -EAGAIN if one preallocated page is required to copy the next PTE. * Otherwise, returns the number of copied PTEs (at least 1). */static inline intcopy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int max_nr, int *rss, struct folio **prealloc){ fpb_t flags = FPB_MERGE_WRITE; struct page *page; struct folio *folio; int err, nr;
page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte;
folio = page_folio(page);
/* * If we likely have to copy, just don't bother with batching. Make * sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) { if (!(src_vma->vm_flags & VM_SHARED)) flags |= FPB_RESPECT_DIRTY; if (vma_soft_dirty_enabled(src_vma)) flags |= FPB_RESPECT_SOFT_DIRTY;
nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); folio_ref_add(folio, nr); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, nr, dst_vma, src_vma))) { folio_ref_sub(folio, nr); return -EAGAIN; } rss[MM_ANONPAGES] += nr; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { folio_dup_file_rmap_ptes(folio, page, nr, dst_vma); rss[mm_counter_file(folio)] += nr; } __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, nr); return nr; }
folio_get(folio); if (folio_test_anon(folio)) { /* * If this page may have been pinned by the parent process, * copy the page immediately for the child so that we'll always * guarantee the pinned page won't be randomly replaced in the * future. */ if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) { /* Page may be pinned, we have to copy. */ folio_put(folio); err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, addr, rss, prealloc, page); return err ? err : 1; } rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { folio_dup_file_rmap_pte(folio, page, dst_vma); rss[mm_counter_file(folio)]++; }
copy_pte: __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1); return 1;}
static inline struct folio *folio_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma, unsigned long addr, bool need_zero){ struct folio *new_folio;
if (need_zero) new_folio = vma_alloc_zeroed_movable_folio(vma, addr); else new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr);
if (!new_folio) return NULL;
if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) { folio_put(new_folio); return NULL; } folio_throttle_swaprate(new_folio, GFP_KERNEL);
return new_folio;}
static intcopy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, unsigned long end){ struct mm_struct *dst_mm = dst_vma->vm_mm; struct mm_struct *src_mm = src_vma->vm_mm; pte_t *orig_src_pte, *orig_dst_pte; pte_t *src_pte, *dst_pte; pmd_t dummy_pmdval; pte_t ptent; spinlock_t *src_ptl, *dst_ptl; int progress, max_nr, ret = 0; int rss[NR_MM_COUNTERS]; softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; int nr;
again: progress = 0; init_rss_vec(rss);
/* * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the * error handling here, assume that exclusive mmap_lock on dst and src * protects anon from unexpected THP transitions; with shmem and file * protected by mmap_lock-less collapse skipping areas with an anon rmap * (whereas vma_needs_copy() skips areas without one). A rework * can remove such assumptions later, but this is good enough for now. */ dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); if (!dst_pte) { ret = -ENOMEM; goto out; }
/* * We already hold the exclusive mmap_lock, the copy_pte_range() and * retract_page_tables() are using vma->anon_vma to be exclusive, so * the PTE page is stable, and there is no need to get pmdval and do * pmd_same() check. */ src_pte = pte_offset_map_rw_nolock(src_mm, src_pmd, addr, &dummy_pmdval, &src_ptl); if (!src_pte) { pte_unmap_unlock(dst_pte, dst_ptl); /* ret == 0 */ goto out; } spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); orig_src_pte = src_pte; orig_dst_pte = dst_pte; lazy_mmu_mode_enable();
do { nr = 1;
/* * We are holding two locks at this point - either of them * could generate latencies in another task on another CPU. */ if (progress >= 32) { progress = 0; if (need_resched() || spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) break; } ptent = ptep_get(src_pte); if (pte_none(ptent)) { progress++; continue; } if (unlikely(!pte_present(ptent))) { ret = copy_nonpresent_pte(dst_mm, src_mm, dst_pte, src_pte, dst_vma, src_vma, addr, rss); if (ret == -EIO) { entry = softleaf_from_pte(ptep_get(src_pte)); break; } else if (ret == -EBUSY) { break; } else if (!ret) { progress += 8; continue; } ptent = ptep_get(src_pte); VM_WARN_ON_ONCE(!pte_present(ptent));
/* * Device exclusive entry restored, continue by copying * the now present pte. */ WARN_ON_ONCE(ret != -ENOENT); } /* copy_present_ptes() will clear `*prealloc' if consumed */ max_nr = (end - addr) / PAGE_SIZE; ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, ptent, addr, max_nr, rss, &prealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. * If copy failed due to hwpoison in source page, break out. */ if (unlikely(ret == -EAGAIN || ret == -EHWPOISON)) break; if (unlikely(prealloc)) { /* * pre-alloc page cannot be reused by next time so as * to strictly follow mempolicy (e.g., alloc_page_vma() * will allocate page according to address). This * could only happen if one pinned pte changed. */ folio_put(prealloc); prealloc = NULL; } nr = ret; progress += 8 * nr; } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, addr != end);
lazy_mmu_mode_disable(); pte_unmap_unlock(orig_src_pte, src_ptl); add_mm_rss_vec(dst_mm, rss); pte_unmap_unlock(orig_dst_pte, dst_ptl); cond_resched();
if (ret == -EIO) { VM_WARN_ON_ONCE(!entry.val); if (swap_retry_table_alloc(entry, GFP_KERNEL) < 0) { ret = -ENOMEM; goto out; } entry.val = 0; } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) { goto out; } else if (ret == -EAGAIN) { prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; } else if (ret < 0) { VM_WARN_ON_ONCE(1); }
/* We've captured and resolved the error. Reset, try again. */ ret = 0;
if (addr != end) goto again;out: if (unlikely(prealloc)) folio_put(prealloc); return ret;}
static inline intcopy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pud_t *dst_pud, pud_t *src_pud, unsigned long addr, unsigned long end){ struct mm_struct *dst_mm = dst_vma->vm_mm; struct mm_struct *src_mm = src_vma->vm_mm; pmd_t *src_pmd, *dst_pmd; unsigned long next;
dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); if (!dst_pmd) return -ENOMEM; src_pmd = pmd_offset(src_pud, addr); do { next = pmd_addr_end(addr, end); if (pmd_is_huge(*src_pmd)) { int err;
VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma); err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd, addr, dst_vma, src_vma); if (err == -ENOMEM) return -ENOMEM; if (!err) continue; /* fall through */ } if (pmd_none_or_clear_bad(src_pmd)) continue; if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd, addr, next)) return -ENOMEM; } while (dst_pmd++, src_pmd++, addr = next, addr != end); return 0;}
static inline intcopy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr, unsigned long end){ struct mm_struct *dst_mm = dst_vma->vm_mm; struct mm_struct *src_mm = src_vma->vm_mm; pud_t *src_pud, *dst_pud; unsigned long next;
dst_pud = pud_alloc(dst_mm, dst_p4d, addr); if (!dst_pud) return -ENOMEM; src_pud = pud_offset(src_p4d, addr); do { next = pud_addr_end(addr, end); if (pud_trans_huge(*src_pud)) { int err;
VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma); err = copy_huge_pud(dst_mm, src_mm, dst_pud, src_pud, addr, src_vma); if (err == -ENOMEM) return -ENOMEM; if (!err) continue; /* fall through */ } if (pud_none_or_clear_bad(src_pud)) continue; if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud, addr, next)) return -ENOMEM; } while (dst_pud++, src_pud++, addr = next, addr != end); return 0;}
static inline intcopy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr, unsigned long end){ struct mm_struct *dst_mm = dst_vma->vm_mm; p4d_t *src_p4d, *dst_p4d; unsigned long next;
dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); if (!dst_p4d) return -ENOMEM; src_p4d = p4d_offset(src_pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(src_p4d)) continue; if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d, addr, next)) return -ENOMEM; } while (dst_p4d++, src_p4d++, addr = next, addr != end); return 0;}
/* * Return true if the vma needs to copy the pgtable during this fork(). Return * false when we can speed up fork() by allowing lazy page faults later until * when the child accesses the memory range. */static boolvma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma){ /* * We check against dst_vma as while sane VMA flags will have been * copied, VM_UFFD_WP may be set only on dst_vma. */ if (dst_vma->vm_flags & VM_COPY_ON_FORK) return true; /* * The presence of an anon rmap indicates the VMA may map anonymous * folios which naturally cannot be reconstituted on page fault. */ if (vma_has_anon_rmap(src_vma)) return true;
/* * Don't copy ptes where a page fault will fill them correctly. Fork * becomes much lighter when there are big shared or private readonly * mappings. The tradeoff is that copy_page_range is more efficient * than faulting. */ return false;}
intcopy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma){ pgd_t *src_pgd, *dst_pgd; unsigned long addr = src_vma->vm_start; unsigned long end = src_vma->vm_end; struct mm_struct *dst_mm = dst_vma->vm_mm; struct mm_struct *src_mm = src_vma->vm_mm; struct mmu_notifier_range range; unsigned long next; bool is_cow; int ret;
if (!vma_needs_copy(dst_vma, src_vma)) return 0;
if (vma_is_hugetlb(src_vma)) return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
/* * We need to invalidate the secondary MMU mappings only when * there could be a permission downgrade on the ptes of the * parent mm. And a permission downgrade will only happen if * vma_is_cow_mapping() returns true. */ is_cow = vma_is_cow_mapping(src_vma);
if (is_cow) { mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0, src_mm, addr, end); mmu_notifier_invalidate_range_start(&range); /* * Disabling preemption is not needed for the write side, as * the read side doesn't spin, but goes to the mmap_lock. * * Use the raw variant of the seqcount_t write API to avoid * lockdep complaining about preemptibility. */ vma_assert_write_locked(src_vma); raw_write_seqcount_begin(&src_mm->write_protect_seq); }
ret = 0; dst_pgd = pgd_offset(dst_mm, addr); src_pgd = pgd_offset(src_mm, addr); do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(src_pgd)) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, addr, next))) { ret = -ENOMEM; break; } } while (dst_pgd++, src_pgd++, addr = next, addr != end);
if (is_cow) { raw_write_seqcount_end(&src_mm->write_protect_seq); mmu_notifier_invalidate_range_end(&range); } return ret;}
/* Whether we should zap all COWed (private) pages too */static inline bool should_zap_cows(struct zap_details *details){ /* By default, zap all pages */ if (!details) return true;
VM_WARN_ON_ONCE(details->skip_cows && details->reclaim_pt);
/* Or, we zap COWed pages only if the caller wants to */ return !details->skip_cows;}
/* Decides whether we should zap this folio with the folio pointer specified */static inline bool should_zap_folio(struct zap_details *details, struct folio *folio){ /* If we can make a decision without *folio.. */ if (should_zap_cows(details)) return true;
/* Otherwise we should only zap non-anon folios */ return !folio_test_anon(folio);}
static inline bool zap_drop_markers(struct zap_details *details){ if (!details) return false;
return details->zap_flags & ZAP_FLAG_DROP_MARKER;}
/** * cond_install_uffd_wp_ptes - install uffd-wp markers after clearing PTEs * @vma: The VMA the pages are mapped into. * @addr: Address the first page of this batch is mapped at. * @ptep: Page table pointer for the first entry of this batch. * @pte: Old value of the entry pointed to by @ptep. * @nr_ptes: Number of entries to install. * * If the PTEs were write-protected by uffd-wp in any form, arm special PTEs * to replace none PTEs. NOTE! This should only be called when the PTEs are * already cleared so we will never accidentally replace something valuable. * Meanwhile none PTEs also mean we are not demoting the PTEs so a TLB flush is * not needed. E.g., when the PTEs were cleared, the caller should have taken * care of the TLB flush. * * Must be called with the page table lock held so that no thread will see the * none PTEs, and if they see them, they'll fault and serialize at the page table * lock. * * Returns true if uffd-wp PTEs were installed, false otherwise. */bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte, unsigned long nr_ptes){ bool arm_uffd_pte = false;
if (!uffd_supports_wp_marker()) return false;
/* The current status of the pte should be "cleared" before calling */ WARN_ON_ONCE(!pte_none(ptep_get(ptep)));
/* * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole * thing, because when zapping either it means it's dropping the * page, or in TTU where the present pte will be quickly replaced * with a swap pte. There's no way of leaking the bit. */ if (vma_is_anonymous(vma) || !userfaultfd_wp(vma)) return false;
/* A uffd-wp wr-protected normal pte */ if (unlikely(pte_present(pte) && pte_uffd(pte))) arm_uffd_pte = true;
/* * A uffd-wp wr-protected swap pte. Note: this should even cover an * existing pte marker with uffd-wp bit set. */ if (unlikely(pte_swp_uffd_any(pte))) arm_uffd_pte = true;
if (likely(!arm_uffd_pte)) return false;
for (;;) { set_pte_at(vma->vm_mm, addr, ptep, make_pte_marker(PTE_MARKER_UFFD_WP)); if (--nr_ptes == 0) break; ptep++; addr += PAGE_SIZE; }
return true;}
/* * This function makes sure that we'll replace the none pte with an uffd-wp * swap special pte marker when necessary. Must be with the pgtable lock held. * * Returns true if uffd-wp ptes was installed, false otherwise. */static inline boolzap_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, int nr, struct zap_details *details, pte_t pteval){ if (zap_drop_markers(details)) return false;
return cond_install_uffd_wp_ptes(vma, addr, pte, pteval, nr);}
static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, struct vm_area_struct *vma, struct folio *folio, struct page *page, pte_t *pte, pte_t ptent, unsigned int nr, unsigned long addr, struct zap_details *details, int *rss, bool *force_flush, bool *force_break, bool *any_skipped){ struct mm_struct *mm = tlb->mm; bool delay_rmap = false;
if (!folio_test_anon(folio)) { ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { delay_rmap = true; *force_flush = true; } } if (pte_young(ptent) && likely(vma_has_recency(vma))) folio_mark_accessed(folio); rss[mm_counter(folio)] -= nr; } else { /* We don't need up-to-date accessed/dirty bits. */ clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); rss[MM_ANONPAGES] -= nr; } /* Checking a single PTE in a batch is sufficient. */ arch_check_zapped_pte(vma, ptent); tlb_remove_tlb_entries(tlb, pte, nr, addr); if (unlikely(userfaultfd_pte_wp(vma, ptent))) *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);
if (!delay_rmap) { folio_remove_rmap_ptes(folio, page, nr, vma);
if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { *force_flush = true; *force_break = true; }}
/* * Zap or skip at least one present PTE, trying to batch-process subsequent * PTEs that map consecutive pages of the same folio. * * Returns the number of processed (skipped or zapped) PTEs (at least 1). */static inline int zap_present_ptes(struct mmu_gather *tlb, struct vm_area_struct *vma, pte_t *pte, pte_t ptent, unsigned int max_nr, unsigned long addr, struct zap_details *details, int *rss, bool *force_flush, bool *force_break, bool *any_skipped){ struct mm_struct *mm = tlb->mm; struct folio *folio; struct page *page; int nr;
page = vm_normal_page(vma, addr, ptent); if (!page) { /* We don't need up-to-date accessed/dirty bits. */ ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm); arch_check_zapped_pte(vma, ptent); tlb_remove_tlb_entry(tlb, pte, addr); if (userfaultfd_pte_wp(vma, ptent)) *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, 1, details, ptent); ksm_might_unmap_zero_page(mm, ptent); return 1; }
folio = page_folio(page); if (unlikely(!should_zap_folio(details, folio))) { *any_skipped = true; return 1; }
/* * Make sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { nr = folio_pte_batch(folio, pte, ptent, max_nr); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, force_break, any_skipped); return nr; } zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr, details, rss, force_flush, force_break, any_skipped); return 1;}
static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, struct vm_area_struct *vma, pte_t *pte, pte_t ptent, unsigned int max_nr, unsigned long addr, struct zap_details *details, int *rss, bool *any_skipped){ softleaf_t entry; int nr = 1;
*any_skipped = true; entry = softleaf_from_pte(ptent); if (softleaf_is_device_private(entry) || softleaf_is_device_exclusive(entry)) { struct page *page = softleaf_to_page(entry); struct folio *folio = page_folio(page);
if (unlikely(!should_zap_folio(details, folio))) return 1; /* * Both device private/exclusive mappings should only * work with anonymous page so far, so we don't need to * consider uffd-wp bit when zap. For more information, * see zap_install_uffd_wp_if_needed(). */ WARN_ON_ONCE(!folio_test_anon(folio)); rss[mm_counter(folio)]--; folio_remove_rmap_pte(folio, page, vma); folio_put(folio); } else if (softleaf_is_swap(entry)) { /* Genuine swap entries, hence a private anon pages */ if (!should_zap_cows(details)) return 1;
nr = swap_pte_batch(pte, max_nr, ptent); rss[MM_SWAPENTS] -= nr; swap_put_entries_direct(entry, nr); } else if (softleaf_is_migration(entry)) { struct folio *folio = softleaf_to_folio(entry);
if (!should_zap_folio(details, folio)) return 1; rss[mm_counter(folio)]--; } else if (softleaf_is_uffd_wp_marker(entry)) { /* * For anon: always drop the marker; for file: only * drop the marker if explicitly requested. */ if (!vma_is_anonymous(vma) && !zap_drop_markers(details)) return 1; } else if (softleaf_is_guard_marker(entry)) { /* * Ordinary zapping should not remove guard PTE * markers. Only do so if we should remove PTE markers * in general. */ if (!zap_drop_markers(details)) return 1; } else if (softleaf_is_hwpoison(entry) || softleaf_is_poison_marker(entry)) { if (!should_zap_cows(details)) return 1; } else { /* We should have covered all the swap entry types */ pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr); *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);
return nr;}
static inline int do_zap_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pte_t *pte, unsigned long addr, unsigned long end, struct zap_details *details, int *rss, bool *force_flush, bool *force_break, bool *any_skipped){ pte_t ptent = ptep_get(pte); int max_nr = (end - addr) / PAGE_SIZE; int nr = 0;
/* Skip all consecutive none ptes */ if (pte_none(ptent)) { for (nr = 1; nr < max_nr; nr++) { ptent = ptep_get(pte + nr); if (!pte_none(ptent)) break; } max_nr -= nr; if (!max_nr) return nr; pte += nr; addr += nr * PAGE_SIZE; }
if (pte_present(ptent)) nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr, details, rss, force_flush, force_break, any_skipped); else nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr, details, rss, any_skipped);
return nr;}
static bool pte_table_reclaim_possible(unsigned long start, unsigned long end, struct zap_details *details){ if (!IS_ENABLED(CONFIG_PT_RECLAIM)) return false; /* Only zap if we are allowed to and cover the full page table. */ return details && details->reclaim_pt && (end - start >= PMD_SIZE);}
static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd, spinlock_t *ptl, pmd_t *pmdval){ spinlock_t *pml = pmd_lockptr(mm, pmd);
if (ptl != pml && !spin_trylock(pml)) return false;
*pmdval = pmdp_get(pmd); pmd_clear(pmd); if (ptl != pml) spin_unlock(pml); return true;}
static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, pmd_t *pmdval){ spinlock_t *pml, *ptl = NULL; pte_t *start_pte, *pte; int i;
pml = pmd_lock(mm, pmd); start_pte = pte_offset_map_rw_nolock(mm, pmd, addr, pmdval, &ptl); if (!start_pte) goto out_ptl; if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);
for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { if (!pte_none(ptep_get(pte))) goto out_ptl; } pte_unmap(start_pte);
pmd_clear(pmd);
if (ptl != pml) spin_unlock(ptl); spin_unlock(pml); return true;out_ptl: if (start_pte) pte_unmap_unlock(start_pte, ptl); if (ptl != pml) spin_unlock(pml); return false;}
static unsigned long zap_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, struct zap_details *details){ bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details); bool force_flush = false, force_break = false; struct mm_struct *mm = tlb->mm; int rss[NR_MM_COUNTERS]; spinlock_t *ptl; pte_t *start_pte; pte_t *pte; pmd_t pmdval; unsigned long start = addr; bool direct_reclaim = true; int nr;
retry: tlb_change_page_size(tlb, PAGE_SIZE); init_rss_vec(rss); start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) return addr;
flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); do { bool any_skipped = false;
if (need_resched()) { direct_reclaim = false; break; }
nr = do_zap_pte_range(tlb, vma, pte, addr, end, details, rss, &force_flush, &force_break, &any_skipped); if (any_skipped) can_reclaim_pt = false; if (unlikely(force_break)) { addr += nr * PAGE_SIZE; direct_reclaim = false; break; } } while (pte += nr, addr += PAGE_SIZE * nr, addr != end);
/* * Fast path: try to hold the pmd lock and unmap the PTE page. * * If the pte lock was released midway (retry case), or if the attempt * to hold the pmd lock failed, then we need to recheck all pte entries * to ensure they are still none, thereby preventing the pte entries * from being repopulated by another thread. */ if (can_reclaim_pt && direct_reclaim && addr == end) direct_reclaim = zap_empty_pte_table(mm, pmd, ptl, &pmdval);
add_mm_rss_vec(mm, rss); lazy_mmu_mode_disable();
/* Do the actual TLB flush before dropping ptl */ if (force_flush) { tlb_flush_mmu_tlbonly(tlb); tlb_flush_rmaps(tlb, vma); } pte_unmap_unlock(start_pte, ptl);
/* * If we forced a TLB flush (either due to running out of * batch buffers or because we needed to flush dirty TLB * entries before releasing the ptl), free the batched * memory too. Come back again if we didn't do everything. */ if (force_flush) tlb_flush_mmu(tlb);
if (addr != end) { cond_resched(); force_flush = false; force_break = false; goto retry; }
if (can_reclaim_pt) { if (direct_reclaim || zap_pte_table_if_empty(mm, pmd, start, &pmdval)) { pte_free_tlb(tlb, pmd_pgtable(pmdval), start); mm_dec_nr_ptes(mm); } }
return addr;}
static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pud, unsigned long addr, unsigned long end, struct zap_details *details){ pmd_t *pmd; unsigned long next;
pmd = pmd_offset(pud, addr); do { next = pmd_addr_end(addr, end); if (pmd_is_huge(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); else if (zap_huge_pmd(tlb, vma, pmd, addr)) { addr = next; continue; } /* fall through */ } else if (details && details->single_folio && folio_test_pmd_mappable(details->single_folio) && next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) { sync_with_folio_pmd_zap(tlb->mm, pmd); } if (pmd_none(*pmd)) { addr = next; continue; } addr = zap_pte_range(tlb, vma, pmd, addr, next, details); if (addr != next) pmd--; } while (pmd++, cond_resched(), addr != end);
return addr;}
static inline unsigned long zap_pud_range(struct mmu_gather *tlb, struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr, unsigned long end, struct zap_details *details){ pud_t *pud; unsigned long next;
pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end); if (pud_trans_huge(*pud)) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); else if (zap_huge_pud(tlb, vma, pud, addr)) goto next; /* fall through */ } if (pud_none_or_clear_bad(pud)) continue; next = zap_pmd_range(tlb, vma, pud, addr, next, details);next: cond_resched(); } while (pud++, addr = next, addr != end);
return addr;}
static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, unsigned long end, struct zap_details *details){ p4d_t *p4d; unsigned long next;
p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(p4d)) continue; next = zap_pud_range(tlb, vma, p4d, addr, next, details); } while (p4d++, addr = next, addr != end);
return addr;}
static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long start, unsigned long end, struct zap_details *details){ const bool reaping = details && details->reaping;
VM_WARN_ON_ONCE(start >= end || !range_in_vma(vma, start, end));
/* uprobe_munmap() might sleep, so skip it when reaping. */ if (vma->vm_file && !reaping) uprobe_munmap(vma, start, end);
if (unlikely(vma_is_hugetlb(vma))) { zap_flags_t zap_flags = details ? details->zap_flags : 0;
VM_WARN_ON_ONCE(reaping); /* * vm_file will be NULL when we fail early while instantiating * a new mapping. In this case, no pages were mapped yet and * there is nothing to do. */ if (!vma->vm_file) return; __unmap_hugepage_range(tlb, vma, start, end, NULL, zap_flags); } else { unsigned long next, addr = start; pgd_t *pgd;
tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; next = zap_p4d_range(tlb, vma, pgd, addr, next, details); } while (pgd++, addr = next, addr != end); tlb_end_vma(tlb, vma); }}
/** * zap_vma_for_reaping - zap all page table entries in the vma without blocking * @vma: The vma to zap. * * Zap all page table entries in the vma without blocking for use by the oom * killer. Hugetlb vmas are not supported. * * Returns: 0 on success, -EBUSY if we would have to block. */int zap_vma_for_reaping(struct vm_area_struct *vma){ struct zap_details details = { .reaping = true, }; struct mmu_notifier_range range; struct mmu_gather tlb;
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, vma->vm_start, vma->vm_end); tlb_gather_mmu(&tlb, vma->vm_mm); if (mmu_notifier_invalidate_range_start_nonblock(&range)) { tlb_finish_mmu(&tlb); return -EBUSY; } __zap_vma_range(&tlb, vma, range.start, range.end, &details); mmu_notifier_invalidate_range_end(&range); tlb_finish_mmu(&tlb); return 0;}
/** * unmap_vmas - unmap a range of memory covered by a list of vma's * @tlb: address of the caller's struct mmu_gather * @unmap: The unmap_desc * * Unmap all pages in the vma list. * * Only addresses between `start' and `end' will be unmapped. * * The VMA list must be sorted in ascending virtual address order. * * unmap_vmas() assumes that the caller will flush the whole unmapped address * range after unmap_vmas() returns. So the only responsibility here is to * ensure that any thus-far unmapped pages are flushed before unmap_vmas() * drops the lock and schedules. */void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap){ struct vm_area_struct *vma; struct mmu_notifier_range range; struct zap_details details = { .zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP, };
vma = unmap->first; mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm, unmap->vma_start, unmap->vma_end); mmu_notifier_invalidate_range_start(&range); do { unsigned long start = max(vma->vm_start, unmap->vma_start); unsigned long end = min(vma->vm_end, unmap->vma_end);
hugetlb_zap_begin(vma, &start, &end); __zap_vma_range(tlb, vma, start, end, &details); hugetlb_zap_end(vma, &details); vma = mas_find(unmap->mas, unmap->tree_end - 1); } while (vma); mmu_notifier_invalidate_range_end(&range);}
/** * zap_vma_range_batched - zap page table entries in a vma range * @tlb: pointer to the caller's struct mmu_gather * @vma: the vma covering the range to zap * @address: starting address of the range to zap * @size: number of bytes to zap * @details: details specifying zapping behavior * * @tlb must not be NULL. The provided address range must be fully * contained within @vma. If @vma is for hugetlb, @tlb is flushed and * re-initialized by this function. * * If @details is NULL, this function will zap all page table entries. */void zap_vma_range_batched(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long address, unsigned long size, struct zap_details *details){ const unsigned long end = address + size; struct mmu_notifier_range range;
VM_WARN_ON_ONCE(!tlb || tlb->mm != vma->vm_mm);
if (unlikely(!size)) return;
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, address, end); hugetlb_zap_begin(vma, &range.start, &range.end); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); /* * unmap 'address-end' not 'range.start-range.end' as range * could have been expanded for hugetlb pmd sharing. */ __zap_vma_range(tlb, vma, address, end, details); mmu_notifier_invalidate_range_end(&range); if (vma_is_hugetlb(vma)) { /* * flush tlb and free resources before hugetlb_zap_end(), to * avoid concurrent page faults' allocation failure. */ tlb_finish_mmu(tlb); hugetlb_zap_end(vma, details); tlb_gather_mmu(tlb, vma->vm_mm); }}
/** * zap_vma_range - zap all page table entries in a vma range * @vma: the vma covering the range to zap * @address: starting address of the range to zap * @size: number of bytes to zap * * The provided address range must be fully contained within @vma. */void zap_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size){ struct mmu_gather tlb;
tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, address, size, NULL); tlb_finish_mmu(&tlb);}
/** * zap_special_vma_range - zap all page table entries in a VMA which is not * mm-managed * @vma: the vma covering the range to zap * @address: starting address of the range to zap * @size: number of bytes to zap * * This function does nothing when the provided address range is not fully * contained in @vma, or when @vma is mm-managed. */void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size){ if (!range_in_vma(vma, address, address + size) || vma_is_mm_managed(vma)) return;
zap_vma_range(vma, address, size);}EXPORT_SYMBOL_GPL(zap_special_vma_range);
static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr){ pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd;
pgd = pgd_offset(mm, addr); p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return NULL; pud = pud_alloc(mm, p4d, addr); if (!pud) return NULL; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return NULL;
VM_BUG_ON(pmd_trans_huge(*pmd)); return pmd;}
pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl){ pmd_t *pmd = walk_to_pmd(mm, addr);
if (!pmd) return NULL; return pte_alloc_map_lock(mm, pmd, addr, ptl);}
static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma){ VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP); /* * Whoever wants to forbid the zeropage after some zeropages * might already have been mapped has to scan the page tables and * bail out on any zeropages. Zeropages in COW mappings can * be unshared using FAULT_FLAG_UNSHARE faults. */ if (mm_forbids_zeropage(vma->vm_mm)) return false; /* zeropages in COW mappings are common and unproblematic. */ if (vma_is_cow_mapping(vma)) return true; /* Mappings that do not allow for writable PTEs are unproblematic. */ if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) return true; /* * Why not allow any VMA that has vm_ops->pfn_mkwrite? GUP could * find the shared zeropage and longterm-pin it, which would * be problematic as soon as the zeropage gets replaced by a different * page due to vma->vm_ops->pfn_mkwrite, because what's mapped would * now differ to what GUP looked up. FSDAX is incompatible to * FOLL_LONGTERM and VM_IO is incompatible to GUP completely (see * check_vma_flags). */ return vma->vm_ops && vma->vm_ops->pfn_mkwrite && (vma_is_fsdax(vma) || vma->vm_flags & VM_IO);}
static int validate_page_before_insert(struct vm_area_struct *vma, struct page *page){ struct folio *folio = page_folio(page);
if (!folio_ref_count(folio)) return -EINVAL; if (unlikely(is_zero_folio(folio))) { if (!vm_mixed_zeropage_allowed(vma)) return -EINVAL; return 0; } if (folio_test_anon(folio) || page_has_type(page)) return -EINVAL; flush_dcache_folio(folio); return 0;}
static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, pgprot_t prot, bool mkwrite){ struct folio *folio = page_folio(page); pte_t pteval = ptep_get(pte);
if (!pte_none(pteval)) { if (!mkwrite) return -EBUSY;
/* see insert_pfn(). */ if (pte_pfn(pteval) != page_to_pfn(page)) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval))); return -EFAULT; } pteval = maybe_mkwrite(pteval, vma); pteval = pte_mkyoung(pteval); if (ptep_set_access_flags(vma, addr, pte, pteval, 1)) update_mmu_cache(vma, addr, pte); return 0; }
/* Ok, finally just insert the thing.. */ pteval = mk_pte(page, prot); if (unlikely(is_zero_folio(folio))) { pteval = pte_mkspecial(pteval); } else { folio_get(folio); pteval = mk_pte(page, prot); if (mkwrite) { pteval = pte_mkyoung(pteval); pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); } inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); folio_add_file_rmap_pte(folio, page, vma); } set_pte_at(vma->vm_mm, addr, pte, pteval); return 0;}
static int insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page, pgprot_t prot, bool mkwrite){ int retval; pte_t *pte; spinlock_t *ptl;
retval = validate_page_before_insert(vma, page); if (retval) goto out; retval = -ENOMEM; pte = get_locked_pte(vma->vm_mm, addr, &ptl); if (!pte) goto out; retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, mkwrite); pte_unmap_unlock(pte, ptl);out: return retval;}
static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, pgprot_t prot){ int err;
err = validate_page_before_insert(vma, page); if (err) return err; return insert_page_into_pte_locked(vma, pte, addr, page, prot, false);}
/* insert_pages() amortizes the cost of spinlock operations * when inserting pages in a loop. */static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot){ pmd_t *pmd = NULL; pte_t *start_pte, *pte; spinlock_t *pte_lock; struct mm_struct *const mm = vma->vm_mm; unsigned long curr_page_idx = 0; unsigned long remaining_pages_total = *num; unsigned long pages_to_write_in_pmd; int err = 0;more: pmd = walk_to_pmd(mm, addr); if (!pmd) { err = -ENOMEM; goto out; }
pages_to_write_in_pmd = min_t(unsigned long, remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
/* Allocate the PTE if necessary; takes PMD lock once only. */ if (pte_alloc(mm, pmd)) { err = -ENOMEM; goto out; }
while (pages_to_write_in_pmd) { int pte_idx = 0; const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); if (!start_pte) { err = -EFAULT; goto out; } for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { err = insert_page_in_batch_locked(vma, pte, addr, pages[curr_page_idx], prot); if (unlikely(err)) { pte_unmap_unlock(start_pte, pte_lock); remaining_pages_total -= pte_idx; goto out; } addr += PAGE_SIZE; ++curr_page_idx; } pte_unmap_unlock(start_pte, pte_lock); pages_to_write_in_pmd -= batch_size; remaining_pages_total -= batch_size; } if (remaining_pages_total) goto more;out: *num = remaining_pages_total; return err;}
/** * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock. * @vma: user vma to map to * @addr: target start user address of these pages * @pages: source kernel pages * @num: in: number of pages to map. out: number of pages that were *not* * mapped. (0 means all pages were successfully mapped). * * Preferred over vm_insert_page() when inserting multiple pages. * * In case of error, we may have mapped a subset of the provided * pages. It is the caller's responsibility to account for this case. * * The same restrictions apply as in vm_insert_page(). */int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num){ const unsigned long nr_pages = *num; const unsigned long end = addr + PAGE_SIZE * nr_pages;
if (!range_in_vma(vma, addr, end)) return -EFAULT; if (!vma_test(vma, VMA_MIXEDMAP_BIT)) { mmap_assert_write_locked(vma->vm_mm); VM_WARN_ON_ONCE(vma_test(vma, VMA_PFNMAP_BIT)); vma_start_write(vma); vma_set_flags(vma, VMA_MIXEDMAP_BIT); } /* Defer page refcount checking till we're about to map that page. */ return insert_pages(vma, addr, pages, num, vma->vm_page_prot);}EXPORT_SYMBOL(vm_insert_pages);
static void __map_kernel_pages_prepare(struct vm_area_desc *desc){ if (vma_desc_test(desc, VMA_MIXEDMAP_BIT)) return;
mmap_assert_write_locked(desc->mm); VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT)); vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT);}
int map_kernel_pages_prepare(struct vm_area_desc *desc){ const struct mmap_action *action = &desc->action; const unsigned long addr = action->map_kernel.start; unsigned long nr_pages, end;
__map_kernel_pages_prepare(desc);
nr_pages = action->map_kernel.nr_pages; end = addr + PAGE_SIZE * nr_pages; if (!range_in_vma_desc(desc, addr, end)) return -EFAULT;
return 0;}
int map_kernel_pages_complete(struct vm_area_struct *vma, struct mmap_action *action){ unsigned long nr_pages;
nr_pages = action->map_kernel.nr_pages; return insert_pages(vma, action->map_kernel.start, action->map_kernel.pages, &nr_pages, vma->vm_page_prot);}
int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc){ const struct mmap_action *action = &desc->action; const struct discontig_kernel_page_ops *ops = action->map_kernel_discontig.ops;
/* At minimum need to be able to get pages. */ if (WARN_ON_ONCE(!ops || !ops->get)) return -EINVAL;
__map_kernel_pages_prepare(desc); return 0;}
static int apply_discontig_action(struct vm_area_struct *vma, struct discontig_kernel_page_state *state){ unsigned long nr_pages = state->__nr_pages; unsigned long addr = state->addr; unsigned long i;
if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGES) return insert_pages(vma, addr, state->__page_arr, &nr_pages, vma->vm_page_prot);
VM_WARN_ON_ONCE(state->action != DISCONTIG_KERNEL_PAGE_MAP_CONTIG_PAGES); /* Contiguous pages - have to iterate through each one. */ for (i = 0; i < nr_pages; i++, addr += PAGE_SIZE) { struct page *page = state->__page + i; int err;
err = insert_page(vma, addr, page, vma->vm_page_prot, /*mkwrite=*/false); if (err) return err; } return 0;}
int map_discontig_kernel_pages_complete(struct vm_area_struct *vma, struct mmap_action *action){ const struct discontig_kernel_page_ops *ops = action->map_kernel_discontig.ops; struct discontig_kernel_page_state state = { .start = vma->vm_start, .end = vma->vm_end, .addr = vma->vm_start, .pgoff = vma->vm_pgoff, .nr_pages_mapped = 0, .nr_pages_remain = vma_pages(vma), .vm_private_data = vma->vm_private_data, .private = action->map_kernel_discontig.init_private, }; int err = 0;
if (ops->init) err = ops->init(vma->vm_private_data, &state.private); if (err) return err;
do { unsigned long end, pgoff_end; unsigned long nr_pages;
/* Default to abort. */ state.action = DISCONTIG_KERNEL_PAGE_ABORT; err = ops->get(&state); if (err || state.action == DISCONTIG_KERNEL_PAGE_ABORT) return err; nr_pages = state.__nr_pages;
if (!nr_pages || nr_pages > state.nr_pages_remain) return -EINVAL; end = state.addr + PAGE_SIZE * nr_pages; pgoff_end = state.pgoff + nr_pages;
err = apply_discontig_action(vma, &state); if (err) return err;
state.addr = end; state.pgoff = pgoff_end; state.nr_pages_mapped += nr_pages; state.nr_pages_remain -= nr_pages; } while (state.addr < vma->vm_end);
return 0;}
/** * vm_insert_page - insert single page into user vma * @vma: user vma to map to * @addr: target user address of this page * @page: source kernel page * * This allows drivers to insert individual pages they've allocated * into a user vma. The zeropage is supported in some VMAs, * see vm_mixed_zeropage_allowed(). * * The page has to be a nice clean _individual_ kernel allocation. * If you allocate a compound page, you need to have marked it as * such (__GFP_COMP), or manually just split the page up yourself * (see split_page()). * * NOTE! Traditionally this was done with "remap_pfn_range()" which * took an arbitrary page protection parameter. This doesn't allow * that. Your vma protection will have to be set up correctly, which * means that if you want a shared writable mapping, you'd better * ask for a shared writable mapping! * * The page does not need to be reserved. * * Usually this function is called from f_op->mmap() handler * under mm->mmap_lock write-lock, so it can change vma->vm_flags. * Caller must set VM_MIXEDMAP on vma if it wants to call this * function from other places, for example from page-fault handler. * * Return: %0 on success, negative error code otherwise. */int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page){ if (addr < vma->vm_start || addr >= vma->vm_end) return -EFAULT; if (!vma_test(vma, VMA_MIXEDMAP_BIT)) { mmap_assert_write_locked(vma->vm_mm); VM_WARN_ON_ONCE(vma_test(vma, VMA_PFNMAP_BIT)); vma_start_write(vma); vma_set_flags(vma, VMA_MIXEDMAP_BIT); } return insert_page(vma, addr, page, vma->vm_page_prot, false);}EXPORT_SYMBOL(vm_insert_page);
/* * __vm_map_pages - maps range of kernel pages into user vma * @vma: user vma to map to * @pages: pointer to array of source kernel pages * @num: number of pages in page array * @offset: user's requested vm_pgoff * * This allows drivers to map range of kernel pages into a user vma. * The zeropage is supported in some VMAs, see * vm_mixed_zeropage_allowed(). * * Return: 0 on success and error code otherwise. */static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num, unsigned long offset){ unsigned long count = vma_pages(vma); unsigned long uaddr = vma->vm_start;
/* Fail if the user requested offset is beyond the end of the object */ if (offset >= num) return -ENXIO;
/* Fail if the user requested size exceeds available object size */ if (count > num - offset) return -ENXIO;
return vm_insert_pages(vma, uaddr, pages + offset, &count);}
/** * vm_map_pages - maps range of kernel pages starts with non zero offset * @vma: user vma to map to * @pages: pointer to array of source kernel pages * @num: number of pages in page array * * Maps an object consisting of @num pages, catering for the user's * requested vm_pgoff * * If we fail to insert any page into the vma, the function will return * immediately leaving any previously inserted pages present. Callers * from the mmap handler may immediately return the error as their caller * will destroy the vma, removing any successfully inserted pages. Other * callers should make their own arrangements for calling unmap_region(). * * Context: Process context. Called by mmap handlers. * Return: 0 on success and error code otherwise. */int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num){ return __vm_map_pages(vma, pages, num, vma_start_pgoff(vma));}EXPORT_SYMBOL(vm_map_pages);
/** * vm_map_pages_zero - map range of kernel pages starts with zero offset * @vma: user vma to map to * @pages: pointer to array of source kernel pages * @num: number of pages in page array * * Similar to vm_map_pages(), except that it explicitly sets the offset * to 0. This function is intended for the drivers that did not consider * vm_pgoff. * * Context: Process context. Called by mmap handlers. * Return: 0 on success and error code otherwise. */int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num){ return __vm_map_pages(vma, pages, num, 0);}EXPORT_SYMBOL(vm_map_pages_zero);
static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t prot, bool mkwrite){ struct mm_struct *mm = vma->vm_mm; pte_t *pte, entry; spinlock_t *ptl;
pte = get_locked_pte(mm, addr, &ptl); if (!pte) return VM_FAULT_OOM; entry = ptep_get(pte); if (!pte_none(entry)) { if (mkwrite) { /* * For read faults on private mappings the PFN passed * in may not match the PFN we have mapped if the * mapped PFN is a writeable COW page. In the mkwrite * case we are creating a writable PTE for a shared * mapping and we expect the PFNs to match. If they * don't match, we are likely racing with block * allocation and mapping invalidation so just skip the * update. */ if (pte_pfn(entry) != pfn) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry))); goto out_unlock; } entry = pte_mkyoung(entry); entry = maybe_mkwrite(pte_mkdirty(entry), vma); if (ptep_set_access_flags(vma, addr, pte, entry, 1)) update_mmu_cache(vma, addr, pte); } goto out_unlock; }
/* Ok, finally just insert the thing.. */ entry = pte_mkspecial(pfn_pte(pfn, prot));
if (mkwrite) { entry = pte_mkyoung(entry); entry = maybe_mkwrite(pte_mkdirty(entry), vma); }
set_pte_at(mm, addr, pte, entry); update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
out_unlock: pte_unmap_unlock(pte, ptl); return VM_FAULT_NOPAGE;}
/** * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot * @vma: user vma to map to * @addr: target user address of this page * @pfn: source kernel pfn * @pgprot: pgprot flags for the inserted page * * This is exactly like vmf_insert_pfn(), except that it allows drivers * to override pgprot on a per-page basis. * * This only makes sense for IO mappings, and it makes no sense for * COW mappings. In general, using multiple vmas is preferable; * vmf_insert_pfn_prot should only be used if using multiple VMAs is * impractical. * * pgprot typically only differs from @vma->vm_page_prot when drivers set * caching- and encryption bits different than those of @vma->vm_page_prot, * because the caching- or encryption mode may not be known at mmap() time. * * This is ok as long as @vma->vm_page_prot is not used by the core vm * to set caching and encryption bits for those vmas (except for COW pages). * This is ensured by core vm only modifying these page table entries using * functions that don't touch caching- or encryption bits, using pte_modify() * if needed. (See for example mprotect()). * * Also when new page-table entries are created, this is only done using the * fault() callback, and never using the value of vma->vm_page_prot, * except for page-table entries that point to anonymous pages as the result * of COW. * * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot){ /* * Technically, architectures with pte_special can avoid all these * restrictions (same for remap_pfn_range). However we would like * consistency in testing and feature parity among all, so we should * try to keep these invariants in place for everybody. */ BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == (VM_PFNMAP|VM_MIXEDMAP)); BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
if (unlikely(is_zero_pfn(pfn))) return VM_FAULT_SIGBUS;
if (addr < vma->vm_start || addr >= vma->vm_end) return VM_FAULT_SIGBUS;
if (!pfn_modify_allowed(pfn, pgprot)) return VM_FAULT_SIGBUS;
pfnmap_setup_cachemode_pfn(pfn, &pgprot);
return insert_pfn(vma, addr, pfn, pgprot, false);}EXPORT_SYMBOL(vmf_insert_pfn_prot);
/** * vmf_insert_pfn - insert single pfn into user vma * @vma: user vma to map to * @addr: target user address of this page * @pfn: source kernel pfn * * Similar to vm_insert_page, this allows drivers to insert individual pages * they've allocated into a user vma. Same comments apply. * * This function should only be called from a vm_ops->fault handler, and * in that case the handler should return the result of this function. * * vma cannot be a COW mapping. * * As this is called only for pages that do not currently exist, we * do not need to flush old virtual caches or the TLB. * * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn){ return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);}EXPORT_SYMBOL(vmf_insert_pfn);
static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite){ if (unlikely(is_zero_pfn(pfn)) && (mkwrite || !vm_mixed_zeropage_allowed(vma))) return false; /* these checks mirror the abort conditions in vm_normal_page */ if (vma->vm_flags & VM_MIXEDMAP) return true; if (is_zero_pfn(pfn)) return true; return false;}
static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, bool mkwrite){ pgprot_t pgprot = vma->vm_page_prot; int err;
if (!vm_mixed_ok(vma, pfn, mkwrite)) return VM_FAULT_SIGBUS;
if (addr < vma->vm_start || addr >= vma->vm_end) return VM_FAULT_SIGBUS;
pfnmap_setup_cachemode_pfn(pfn, &pgprot);
if (!pfn_modify_allowed(pfn, pgprot)) return VM_FAULT_SIGBUS;
/* * If we don't have pte special, then we have to use the pfn_valid() * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* * refcount the page if pfn_valid is true (hence insert_page rather * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP * without pte special, it would there be refcounted as a normal page. */ if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pfn_valid(pfn)) { struct page *page;
/* * At this point we are committed to insert_page() * regardless of whether the caller specified flags that * result in pfn_t_has_page() == false. */ page = pfn_to_page(pfn); err = insert_page(vma, addr, page, pgprot, mkwrite); } else { return insert_pfn(vma, addr, pfn, pgprot, mkwrite); }
if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;}
vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write){ pgprot_t pgprot = vmf->vma->vm_page_prot; unsigned long addr = vmf->address; int err;
if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) return VM_FAULT_SIGBUS;
err = insert_page(vmf->vma, addr, page, pgprot, write); if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;}EXPORT_SYMBOL_GPL(vmf_insert_page_mkwrite);
vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn){ return __vm_insert_mixed(vma, addr, pfn, false);}EXPORT_SYMBOL(vmf_insert_mixed);
/* * If the insertion of PTE failed because someone else already added a * different entry in the mean time, we treat that as success as we assume * the same entry was actually inserted. */vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn){ return __vm_insert_mixed(vma, addr, pfn, true);}
/* * maps a range of physical memory into the requested pages. the old * mappings are removed. any references to nonexistent pages results * in null mappings (currently treated as "copy-on-access") */static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot){ pte_t *pte, *mapped_pte; spinlock_t *ptl; int err = 0;
mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte) return -ENOMEM; lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); if (!pfn_modify_allowed(pfn, prot)) { err = -EACCES; break; } set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); pfn++; } while (pte++, addr += PAGE_SIZE, addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err;}
static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot){ pmd_t *pmd; unsigned long next; int err;
pfn -= addr >> PAGE_SHIFT; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; VM_BUG_ON(pmd_trans_huge(*pmd)); do { next = pmd_addr_end(addr, end); err = remap_pte_range(mm, pmd, addr, next, pfn + (addr >> PAGE_SHIFT), prot); if (err) return err; } while (pmd++, addr = next, addr != end); return 0;}
static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot){ pud_t *pud; unsigned long next; int err;
pfn -= addr >> PAGE_SHIFT; pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); err = remap_pmd_range(mm, pud, addr, next, pfn + (addr >> PAGE_SHIFT), prot); if (err) return err; } while (pud++, addr = next, addr != end); return 0;}
static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot){ p4d_t *p4d; unsigned long next; int err;
pfn -= addr >> PAGE_SHIFT; p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end); err = remap_pud_range(mm, p4d, addr, next, pfn + (addr >> PAGE_SHIFT), prot); if (err) return err; } while (p4d++, addr = next, addr != end); return 0;}
static int get_remap_pgoff(bool is_cow, unsigned long addr, unsigned long end, unsigned long vm_start, unsigned long vm_end, unsigned long pfn, pgoff_t *vm_pgoff_p){ /* * There's a horrible special case to handle copy-on-write * behaviour that some programs depend on. We mark the "original" * un-COW'ed pages by matching them up with "vma->vm_pgoff". * See vm_normal_page() for details. */ if (is_cow) { if (addr != vm_start || end != vm_end) return -EINVAL; *vm_pgoff_p = pfn; }
return 0;}
static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot){ pgd_t *pgd; unsigned long next; unsigned long end = addr + PAGE_ALIGN(size); struct mm_struct *mm = vma->vm_mm; int err;
if (WARN_ON_ONCE(!PAGE_ALIGNED(addr))) return -EINVAL;
VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS));
BUG_ON(addr >= end); pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); do { next = pgd_addr_end(addr, end); err = remap_p4d_range(mm, pgd, addr, next, pfn + (addr >> PAGE_SHIFT), prot); if (err) return err; } while (pgd++, addr = next, addr != end);
return 0;}
/* * Variant of remap_pfn_range that does not call track_pfn_remap. The caller * must have pre-validated the caching bits of the pgprot_t. */static int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot){ int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);
if (!error) return 0;
/* * A partial pfn range mapping is dangerous: it does not * maintain page reference counts, and callers may free * pages due to the error. So zap it early. */ zap_vma_range(vma, addr, size); return error;}
#ifdef __HAVE_PFNMAP_TRACKINGstatic inline struct pfnmap_track_ctx *pfnmap_track_ctx_alloc(unsigned long pfn, unsigned long size, pgprot_t *prot){ struct pfnmap_track_ctx *ctx;
if (pfnmap_track(pfn, size, prot)) return ERR_PTR(-EINVAL);
ctx = kmalloc_obj(*ctx); if (unlikely(!ctx)) { pfnmap_untrack(pfn, size); return ERR_PTR(-ENOMEM); }
ctx->pfn = pfn; ctx->size = size; kref_init(&ctx->kref); return ctx;}
void pfnmap_track_ctx_release(struct kref *ref){ struct pfnmap_track_ctx *ctx = container_of(ref, struct pfnmap_track_ctx, kref);
pfnmap_untrack(ctx->pfn, ctx->size); kfree(ctx);}
static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot){ struct pfnmap_track_ctx *ctx = NULL; int err;
size = PAGE_ALIGN(size);
/* * If we cover the full VMA, we'll perform actual tracking, and * remember to untrack when the last reference to our tracking * context from a VMA goes away. We'll keep tracking the whole pfn * range even during VMA splits and partial unmapping. * * If we only cover parts of the VMA, we'll only setup the cachemode * in the pgprot for the pfn range. */ if (addr == vma->vm_start && addr + size == vma->vm_end) { if (vma->pfnmap_track_ctx) return -EINVAL; ctx = pfnmap_track_ctx_alloc(pfn, size, &prot); if (IS_ERR(ctx)) return PTR_ERR(ctx); } else if (pfnmap_setup_cachemode(pfn, size, &prot)) { return -EINVAL; }
err = remap_pfn_range_notrack(vma, addr, pfn, size, prot); if (ctx) { if (err) kref_put(&ctx->kref, pfnmap_track_ctx_release); else vma->pfnmap_track_ctx = ctx; } return err;}
static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot){ return remap_pfn_range_track(vma, addr, pfn, size, prot);}#elsestatic int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot){ return remap_pfn_range_notrack(vma, addr, pfn, size, prot);}#endif
int remap_pfn_range_prepare(struct vm_area_desc *desc){ const struct mmap_action *action = &desc->action; const unsigned long start = action->remap.start; const unsigned long end = start + action->remap.size; const unsigned long pfn = action->remap.start_pfn; const bool is_cow = vma_desc_is_cow_mapping(desc); int err;
if (!range_in_vma_desc(desc, start, end)) return -EFAULT;
err = get_remap_pgoff(is_cow, start, end, desc->start, desc->end, pfn, &desc->pgoff); if (err) return err;
vma_desc_set_flags_mask(desc, VMA_REMAP_FLAGS); return 0;}
static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size){ const unsigned long end = addr + PAGE_ALIGN(size); const bool is_cow = vma_is_cow_mapping(vma); int err;
err = get_remap_pgoff(is_cow, addr, end, vma->vm_start, vma->vm_end, pfn, &vma->vm_pgoff); if (err) return err;
vma_set_flags_mask(vma, VMA_REMAP_FLAGS); return 0;}
/** * remap_pfn_range - remap kernel memory to userspace * @vma: user vma to map to * @addr: target page aligned user address to start at * @pfn: page frame number of kernel physical memory address * @size: size of mapping area * @prot: page protection flags for this mapping * * Note: this is only safe if the mm semaphore is held when called. * * Return: %0 on success, negative error code otherwise. */int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot){ int err;
err = remap_pfn_range_prepare_vma(vma, addr, pfn, size); if (err) return err;
return do_remap_pfn_range(vma, addr, pfn, size, prot);}EXPORT_SYMBOL(remap_pfn_range);
int remap_pfn_range_complete(struct vm_area_struct *vma, struct mmap_action *action){ const unsigned long start = action->remap.start; const unsigned long pfn = action->remap.start_pfn; const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot;
return do_remap_pfn_range(vma, start, pfn, size, prot);}
static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, phys_addr_t start_phys, unsigned long size, unsigned long *pfnp){ unsigned long pfn, pages;
/* Check that the physical memory area passed in looks valid */ if (start_phys + size < start_phys) return -EINVAL; /* * You *really* shouldn't map things that aren't page-aligned, * but we've historically allowed it because IO memory might * just have smaller alignment. */ size += start_phys & ~PAGE_MASK; pfn = start_phys >> PAGE_SHIFT; pages = (size + ~PAGE_MASK) >> PAGE_SHIFT; if (pfn + pages < pfn) return -EINVAL;
/* We start the mapping 'vm_pgoff' pages into the area */ if (vm_pgoff > pages) return -EINVAL; pfn += vm_pgoff; pages -= vm_pgoff;
/* Can we fit all of the mapping? */ if ((vm_len >> PAGE_SHIFT) > pages) return -EINVAL;
*pfnp = pfn; return 0;}
int simple_ioremap_prepare(struct vm_area_desc *desc){ struct mmap_action *action = &desc->action; const phys_addr_t start = action->simple_ioremap.start_phys_addr; const unsigned long size = action->simple_ioremap.size; unsigned long pfn; int err;
err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff, start, size, &pfn); if (err) return err;
/* The I/O remap logic does the heavy lifting. */ mmap_action_ioremap_full(desc, pfn); return io_remap_pfn_range_prepare(desc);}
/** * vm_iomap_memory - remap memory to userspace * @vma: user vma to map to * @start: start of the physical memory to be mapped * @len: size of area * * This is a simplified io_remap_pfn_range() for common driver use. The * driver just needs to give us the physical memory range to be mapped, * we'll figure out the rest from the vma information. * * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get * whatever write-combining details or similar. * * Return: %0 on success, negative error code otherwise. */int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len){ const unsigned long vm_start = vma->vm_start; const unsigned long vm_end = vma->vm_end; const unsigned long vm_len = vm_end - vm_start; unsigned long pfn; int err;
err = __simple_ioremap_prep(vm_len, vma_start_pgoff(vma), start, len, &pfn); if (err) return err;
/* Ok, let it rip */ return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);}EXPORT_SYMBOL(vm_iomap_memory);
static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask){ pte_t *pte, *mapped_pte; int err = 0; spinlock_t *ptl;
if (create) { mapped_pte = pte = (mm == &init_mm) ? pte_alloc_kernel_track(pmd, addr, mask) : pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte) return -ENOMEM; } else { mapped_pte = pte = (mm == &init_mm) ? pte_offset_kernel(pmd, addr) : pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) return -EINVAL; }
lazy_mmu_mode_enable();
if (fn) { do { if (create || !pte_none(ptep_get(pte))) { err = fn(pte, addr, data); if (err) break; } } while (pte++, addr += PAGE_SIZE, addr != end); } *mask |= PGTBL_PTE_MODIFIED;
lazy_mmu_mode_disable();
if (mm != &init_mm) pte_unmap_unlock(mapped_pte, ptl); return err;}
static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask){ pmd_t *pmd; unsigned long next; int err = 0;
BUG_ON(pud_leaf(*pud));
if (create) { pmd = pmd_alloc_track(mm, pud, addr, mask); if (!pmd) return -ENOMEM; } else { pmd = pmd_offset(pud, addr); } do { next = pmd_addr_end(addr, end); if (pmd_none(*pmd) && !create) continue; if (WARN_ON_ONCE(pmd_leaf(*pmd))) return -EINVAL; if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) { if (!create) continue; pmd_clear_bad(pmd); } err = apply_to_pte_range(mm, pmd, addr, next, fn, data, create, mask); if (err) break; } while (pmd++, addr = next, addr != end);
return err;}
static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask){ pud_t *pud; unsigned long next; int err = 0;
if (create) { pud = pud_alloc_track(mm, p4d, addr, mask); if (!pud) return -ENOMEM; } else { pud = pud_offset(p4d, addr); } do { next = pud_addr_end(addr, end); if (pud_none(*pud) && !create) continue; if (WARN_ON_ONCE(pud_leaf(*pud))) return -EINVAL; if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) { if (!create) continue; pud_clear_bad(pud); } err = apply_to_pmd_range(mm, pud, addr, next, fn, data, create, mask); if (err) break; } while (pud++, addr = next, addr != end);
return err;}
static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask){ p4d_t *p4d; unsigned long next; int err = 0;
if (create) { p4d = p4d_alloc_track(mm, pgd, addr, mask); if (!p4d) return -ENOMEM; } else { p4d = p4d_offset(pgd, addr); } do { next = p4d_addr_end(addr, end); if (p4d_none(*p4d) && !create) continue; if (WARN_ON_ONCE(p4d_leaf(*p4d))) return -EINVAL; if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) { if (!create) continue; p4d_clear_bad(p4d); } err = apply_to_pud_range(mm, p4d, addr, next, fn, data, create, mask); if (err) break; } while (p4d++, addr = next, addr != end);
return err;}
static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr, unsigned long size, pte_fn_t fn, void *data, bool create){ pgd_t *pgd; unsigned long start = addr, next; unsigned long end = addr + size; pgtbl_mod_mask mask = 0; int err = 0;
if (WARN_ON(addr >= end)) return -EINVAL;
pgd = pgd_offset(mm, addr); do { next = pgd_addr_end(addr, end); if (pgd_none(*pgd) && !create) continue; if (WARN_ON_ONCE(pgd_leaf(*pgd))) { err = -EINVAL; break; } if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) { if (!create) continue; pgd_clear_bad(pgd); } err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create, &mask); if (err) break; } while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK) arch_sync_kernel_mappings(start, start + size);
return err;}
/* * Scan a region of virtual memory, filling in page tables as necessary * and calling a provided function on each leaf page table. */int apply_to_page_range(struct mm_struct *mm, unsigned long addr, unsigned long size, pte_fn_t fn, void *data){ return __apply_to_page_range(mm, addr, size, fn, data, true);}EXPORT_SYMBOL_GPL(apply_to_page_range);
/* * Scan a region of virtual memory, calling a provided function on * each leaf page table where it exists. * * Unlike apply_to_page_range, this does _not_ fill in page tables * where they are absent. */int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr, unsigned long size, pte_fn_t fn, void *data){ return __apply_to_page_range(mm, addr, size, fn, data, false);}
/* * handle_pte_fault chooses page fault handler according to an entry which was * read non-atomically. Before making any commitment, on those architectures * or configurations (e.g. i386 with PAE) which might give a mix of unmatched * parts, do_swap_page must check under lock before unmapping the pte and * proceeding (but do_wp_page is only called after already making such a check; * and do_anonymous_page can safely check later on). */static inline int pte_unmap_same(struct vm_fault *vmf){ int same = 1;#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION) if (sizeof(pte_t) > sizeof(unsigned long)) { spin_lock(vmf->ptl); same = pte_same(ptep_get(vmf->pte), vmf->orig_pte); spin_unlock(vmf->ptl); }#endif pte_unmap(vmf->pte); vmf->pte = NULL; return same;}
/* * Return: * 0: copied succeeded * -EHWPOISON: copy failed due to hwpoison in source page * -EAGAIN: copied failed (some other reason) */static inline int __wp_page_copy_user(struct page *dst, struct page *src, struct vm_fault *vmf){ int ret; void *kaddr; void __user *uaddr; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; unsigned long addr = vmf->address;
if (likely(src)) { if (copy_mc_user_highpage(dst, src, addr, vma)) return -EHWPOISON; return 0; }
/* * If the source page was a PFN mapping, we don't have * a "struct page" for it. We do a best-effort copy by * just copying from the original user address. If that * fails, we just zero-fill it. Live with it. */ kaddr = kmap_local_page(dst); pagefault_disable(); uaddr = (void __user *)(addr & PAGE_MASK);
/* * On architectures with software "accessed" bits, we would * take a double page fault, so mark it accessed here. */ vmf->pte = NULL; if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { pte_t entry;
vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { /* * Other thread has already handled the fault * and update local tlb only */ if (vmf->pte) update_mmu_tlb(vma, addr, vmf->pte); ret = -EAGAIN; goto pte_unlock; }
entry = pte_mkyoung(vmf->orig_pte); if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0)) update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1); }
/* * This really shouldn't fail, because the page is there * in the page tables. But it might just be unreadable, * in which case we just give up and fill the result with * zeroes. */ if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) { if (vmf->pte) goto warn;
/* Re-validate under PTL if the page is still mapped */ vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { /* The PTE changed under us, update local tlb */ if (vmf->pte) update_mmu_tlb(vma, addr, vmf->pte); ret = -EAGAIN; goto pte_unlock; }
/* * The same page can be mapped back since last copy attempt. * Try to copy again under PTL. */ if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) { /* * Give a warn in case there can be some obscure * use-case */warn: WARN_ON_ONCE(1); clear_page(kaddr); } }
ret = 0;
pte_unlock: if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl); pagefault_enable(); kunmap_local(kaddr); flush_dcache_page(dst);
return ret;}
static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma){ struct file *vm_file = vma->vm_file;
if (vm_file) return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
/* * Special mappings (e.g. VDSO) do not have any file so fake * a default GFP_KERNEL for them. */ return GFP_KERNEL;}
/* * Notify the address space that the page is about to become writable so that * it can prohibit this or wait for the page to get into an appropriate state. * * We do this without the lock held, so that it can sleep if it needs to. */static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio){ vm_fault_t ret; unsigned int old_flags = vmf->flags;
vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
if (vmf->vma->vm_file && IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host)) return VM_FAULT_SIGBUS;
ret = vmf->vma->vm_ops->page_mkwrite(vmf); /* Restore original flags so that caller is not surprised */ vmf->flags = old_flags; if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) return ret; if (unlikely(!(ret & VM_FAULT_LOCKED))) { folio_lock(folio); if (!folio->mapping) { folio_unlock(folio); return 0; /* retry */ } ret |= VM_FAULT_LOCKED; } else VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); return ret;}
/* * Handle dirtying of a page in shared file mapping on a write fault. * * The function expects the page to be locked and unlocks it. */static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct address_space *mapping; struct folio *folio = page_folio(vmf->page); bool dirtied; bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
dirtied = folio_mark_dirty(folio); VM_BUG_ON_FOLIO(folio_test_anon(folio), folio); /* * Take a local copy of the address_space - folio.mapping may be zeroed * by truncate after folio_unlock(). The address_space itself remains * pinned by vma->vm_file's reference. We rely on folio_unlock()'s * release semantics to prevent the compiler from undoing this copying. */ mapping = folio_raw_mapping(folio); folio_unlock(folio);
if (!page_mkwrite) file_update_time(vma->vm_file);
/* * Throttle page dirtying rate down to writeback speed. * * mapping may be NULL here because some device drivers do not * set page.mapping but still dirty their pages * * Drop the mmap_lock before waiting on IO, if we can. The file * is pinning the mapping, as per above. */ if ((dirtied || page_mkwrite) && mapping) { struct file *fpin;
fpin = maybe_unlock_mmap_for_io(vmf, NULL); balance_dirty_pages_ratelimited(mapping); if (fpin) { fput(fpin); return VM_FAULT_COMPLETED; } }
return 0;}
/* * Handle write page faults for pages that can be reused in the current vma * * This can happen either due to the mapping being with the VM_SHARED flag, * or due to us being the last reference standing to the page. In either * case, all we need to do here is to mark the page as writable and update * any related book-keeping. */static inline void wp_page_reuse(struct vm_fault *vmf, struct folio *folio) __releases(vmf->ptl){ struct vm_area_struct *vma = vmf->vma; pte_t entry;
VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE)); VM_WARN_ON(is_zero_pfn(pte_pfn(vmf->orig_pte)));
if (folio) { VM_BUG_ON(folio_test_anon(folio) && !PageAnonExclusive(vmf->page)); /* * Clear the folio's cpupid information as the existing * information potentially belongs to a now completely * unrelated process. */ folio_xchg_last_cpupid(folio, (1 << LAST_CPUPID_SHIFT) - 1); }
flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); entry = pte_mkyoung(vmf->orig_pte); entry = maybe_mkwrite(pte_mkdirty(entry), vma); if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1)) update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); pte_unmap_unlock(vmf->pte, vmf->ptl); count_vm_event(PGREUSE);}
/* * We could add a bitflag somewhere, but for now, we know that all * vm_ops that have a ->map_pages have been audited and don't need * the mmap_lock to be held. */static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma;
if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK)) return 0; vma_end_read(vma); return VM_FAULT_RETRY;}
/** * __vmf_anon_prepare - Prepare to handle an anonymous fault. * @vmf: The vm_fault descriptor passed from the fault handler. * * When preparing to insert an anonymous page into a VMA from a * fault handler, call this function rather than anon_vma_prepare(). * If this vma does not already have an anon rmap and we are * only protected by the per-VMA lock, the caller must retry with the * mmap_lock held. __anon_vma_prepare() will look at adjacent VMAs to * determine if this VMA can share its anon rmap, and that's not safe to * do with only the per-VMA lock held for this VMA. * * Return: 0 if fault handling can proceed. Any other value should be * returned to the caller. */vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; vm_fault_t ret = 0;
if (likely(vma_has_anon_rmap(vma))) return 0; if (vmf->flags & FAULT_FLAG_VMA_LOCK) { if (!mmap_read_trylock(vma->vm_mm)) return VM_FAULT_RETRY; } if (__anon_vma_prepare(vma)) ret = VM_FAULT_OOM; if (vmf->flags & FAULT_FLAG_VMA_LOCK) mmap_read_unlock(vma->vm_mm); return ret;}
/* * Handle the case of a page which we actually need to copy to a new page, * either due to COW or unsharing. * * Called with either the VMA lock or the mmap_lock held (see FAULT_FLAG_VMA_LOCK) * and the old page referenced, but without the ptl held. * * High level logic flow: * * - Allocate a page, copy the content of the old page to the new one. * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc. * - Take the PTL. If the pte changed, bail out and release the allocated page * - If the pte is still the way we remember it, update the page table and all * relevant references. This includes dropping the reference the page-table * held to the old page, as well as updating the rmap. * - In any case, unlock the PTL and drop the reference we took to the old page. */static vm_fault_t wp_page_copy(struct vm_fault *vmf){ const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct folio *old_folio = NULL; struct folio *new_folio = NULL; pte_t entry; int page_copied = 0; struct mmu_notifier_range range; vm_fault_t ret; bool pfn_is_zero;
delayacct_wpcopy_start();
if (vmf->page) old_folio = page_folio(vmf->page); ret = vmf_anon_prepare(vmf); if (unlikely(ret)) goto out;
pfn_is_zero = is_zero_pfn(pte_pfn(vmf->orig_pte)); new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero); if (!new_folio) goto oom;
if (!pfn_is_zero) { int err;
err = __wp_page_copy_user(&new_folio->page, vmf->page, vmf); if (err) { /* * COW failed, if the fault was solved by other, * it's fine. If not, userspace would re-fault on * the same address and we will handle the fault * from the second attempt. * The -EHWPOISON case will not be retried. */ folio_put(new_folio); if (old_folio) folio_put(old_folio);
delayacct_wpcopy_end(); return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } kmsan_copy_page_meta(&new_folio->page, vmf->page); }
__folio_mark_uptodate(new_folio);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address & PAGE_MASK, (vmf->address & PAGE_MASK) + PAGE_SIZE); mmu_notifier_invalidate_range_start(&range);
/* * Re-check the pte - we dropped the lock */ vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { if (old_folio) { if (!folio_test_anon(old_folio)) { dec_mm_counter(mm, mm_counter_file(old_folio)); inc_mm_counter(mm, MM_ANONPAGES); } } else { ksm_might_unmap_zero_page(mm, vmf->orig_pte); inc_mm_counter(mm, MM_ANONPAGES); } flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); entry = folio_mk_pte(new_folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); if (unlikely(unshare)) { if (pte_soft_dirty(vmf->orig_pte)) entry = pte_mksoft_dirty(entry); if (pte_uffd(vmf->orig_pte)) entry = pte_mkuffd(entry); } else { entry = maybe_mkwrite(pte_mkdirty(entry), vma); }
/* * Clear the pte entry and flush it first, before updating the * pte with the new entry, to keep TLBs on different CPUs in * sync. This code used to set the new PTE then flush TLBs, but * that left a window where the new PTE could be loaded into * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); if (old_folio) { /* * Only after switching the pte to the new page may * we remove the mapcount here. Otherwise another * process may come and find the rmap count decremented * before the pte is switched to the new page, and * "reuse" the old page writing into it while our pte * here still points into it and can be read by other * threads. * * The critical issue is to order this * folio_remove_rmap_pte() with the ptp_clear_flush * above. Those stores are ordered by (if nothing else,) * the barrier present in the atomic_add_negative * in folio_remove_rmap_pte(); * * Then the TLB flush in ptep_clear_flush ensures that * no process can access the old page before the * decremented mapcount is visible. And the old page * cannot be reused until after the decremented * mapcount is visible. So transitively, TLBs to * old page will be flushed before it can be reused. */ folio_remove_rmap_pte(old_folio, vmf->page, vma); }
/* Free the old page.. */ new_folio = old_folio; page_copied = 1; pte_unmap_unlock(vmf->pte, vmf->ptl); } else if (vmf->pte) { update_mmu_tlb(vma, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); }
mmu_notifier_invalidate_range_end(&range);
if (new_folio) folio_put(new_folio); if (old_folio) { if (page_copied) free_swap_cache(old_folio); folio_put(old_folio); }
delayacct_wpcopy_end(); return 0;oom: ret = VM_FAULT_OOM;out: if (old_folio) folio_put(old_folio);
delayacct_wpcopy_end(); return ret;}
/** * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE * writeable once the page is prepared * * @vmf: structure describing the fault * @folio: the folio of vmf->page * * This function handles all that is needed to finish a write page fault in a * shared mapping due to PTE being read-only once the mapped page is prepared. * It handles locking of PTE and modifying it. * * The function expects the page to be locked or other protection against * concurrent faults / writeback (such as DAX radix tree locks). * * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before * we acquired PTE lock. */static vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf, struct folio *folio){ WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED)); vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) return VM_FAULT_NOPAGE; /* * We might have raced with another page fault while we released the * pte_offset_map_lock. */ if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); return VM_FAULT_NOPAGE; } wp_page_reuse(vmf, folio); return 0;}
/* * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED * mapping */static vm_fault_t wp_pfn_shared(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma;
if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) { vm_fault_t ret;
pte_unmap_unlock(vmf->pte, vmf->ptl); ret = vmf_can_call_fault(vmf); if (ret) return ret;
vmf->flags |= FAULT_FLAG_MKWRITE; ret = vma->vm_ops->pfn_mkwrite(vmf); if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)) return ret; return finish_mkwrite_fault(vmf, NULL); } wp_page_reuse(vmf, NULL); return 0;}
static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio) __releases(vmf->ptl){ struct vm_area_struct *vma = vmf->vma; vm_fault_t ret = 0;
folio_get(folio);
if (vma->vm_ops && vma->vm_ops->page_mkwrite) { vm_fault_t tmp;
pte_unmap_unlock(vmf->pte, vmf->ptl); tmp = vmf_can_call_fault(vmf); if (tmp) { folio_put(folio); return tmp; }
tmp = do_page_mkwrite(vmf, folio); if (unlikely(!tmp || (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { folio_put(folio); return tmp; } tmp = finish_mkwrite_fault(vmf, folio); if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { folio_unlock(folio); folio_put(folio); return tmp; } } else { wp_page_reuse(vmf, folio); folio_lock(folio); } ret |= fault_dirty_shared_page(vmf); folio_put(folio);
return ret;}
#ifdef CONFIG_TRANSPARENT_HUGEPAGEstatic bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma){ bool exclusive = false;
/* Let's just free up a large folio if only a single page is mapped. */ if (folio_large_mapcount(folio) <= 1) return false;
/* * The assumption for anonymous folios is that each page can only get * mapped once into each MM. The only exception are KSM folios, which * are always small. * * Each taken mapcount must be paired with exactly one taken reference, * whereby the refcount must be incremented before the mapcount when * mapping a page, and the refcount must be decremented after the * mapcount when unmapping a page. * * If all folio references are from mappings, and all mappings are in * the page tables of this MM, then this folio is exclusive to this MM. */ if (test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids)) return false;
VM_WARN_ON_ONCE(folio_test_ksm(folio));
if (unlikely(folio_test_swapcache(folio))) { /* * Note: freeing up the swapcache will fail if some PTEs are * still swap entries. */ if (!folio_trylock(folio)) return false; folio_free_swap(folio); folio_unlock(folio); }
if (folio_large_mapcount(folio) != folio_ref_count(folio)) return false;
/* Stabilize the mapcount vs. refcount and recheck. */ folio_lock_large_mapcount(folio); VM_WARN_ON_ONCE_FOLIO(folio_large_mapcount(folio) > folio_ref_count(folio), folio);
if (test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids)) goto unlock; if (folio_large_mapcount(folio) != folio_ref_count(folio)) goto unlock;
VM_WARN_ON_ONCE_FOLIO(folio_large_mapcount(folio) > folio_nr_pages(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_entire_mapcount(folio), folio); VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != vma->vm_mm->mm_id && folio_mm_id(folio, 1) != vma->vm_mm->mm_id);
/* * Do we need the folio lock? Likely not. If there would have been * references from page migration/swapout, we would have detected * an additional folio reference and never ended up here. */ exclusive = true;unlock: folio_unlock_large_mapcount(folio); return exclusive;}#else /* !CONFIG_TRANSPARENT_HUGEPAGE */static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma){ BUILD_BUG();}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma){ const bool maybe_in_lru_cache = !folio_test_lru(folio); const bool in_swapcache = folio_test_swapcache(folio);
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma);
/* * We have to verify under folio lock: these early checks are * just an optimization to avoid locking the folio and freeing * the swapcache if there is little hope that we can reuse. * * KSM doesn't necessarily raise the folio refcount. */ if (folio_test_ksm(folio) || folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache) return false; if (maybe_in_lru_cache) /* * We cannot easily detect+handle references from * remote LRU caches or references to LRU folios. */ lru_add_drain(); if (folio_ref_count(folio) > 1 + in_swapcache) return false; if (!folio_trylock(folio)) return false; if (folio_test_swapcache(folio)) folio_free_swap(folio); if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) { folio_unlock(folio); return false; } /* * Ok, we've got the only folio reference from our mapping * and the folio is locked, it's dark out, and we're wearing * sunglasses. Hit it. */ folio_move_anon_rmap(folio, vma); folio_unlock(folio); return true;}
/* * This routine handles present pages, when * * users try to write to a shared page (FAULT_FLAG_WRITE) * * GUP wants to take a R/O pin on a possibly shared anonymous page * (FAULT_FLAG_UNSHARE) * * It is done by copying the page to a new address and decrementing the * shared-page counter for the old page. * * Note that this routine assumes that the protection checks have been * done by the caller (the low-level page fault routine in most cases). * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've * done any necessary COW. * * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even * though the page will change only once the write actually happens. This * avoids a few races, and potentially makes it more efficient. * * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK) and pte both mapped and locked. We return with * the same lock still held, but pte unmapped and unlocked. */static vm_fault_t do_wp_page(struct vm_fault *vmf) __releases(vmf->ptl){ const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; struct folio *folio = NULL; pte_t pte;
if (likely(!unshare)) { if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { if (!userfaultfd_wp_async(vma)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return handle_userfault(vmf, VM_UFFD_WP); }
/* * Nothing needed (cache flush, TLB invalidations, * etc.) because we're only removing the uffd-wp bit, * which is completely invisible to the user. */ pte = pte_clear_uffd(ptep_get(vmf->pte));
set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); /* * Update this to be prepared for following up CoW * handling */ vmf->orig_pte = pte; }
/* * Userfaultfd write-protect can defer flushes. Ensure the TLB * is flushed in this case before copying. */ if (unlikely(userfaultfd_wp(vmf->vma) && mm_tlb_flush_pending(vmf->vma->vm_mm))) flush_tlb_page(vmf->vma, vmf->address); }
vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
if (vmf->page) folio = page_folio(vmf->page);
/* * Shared mapping: we are guaranteed to have VM_WRITE and * FAULT_FLAG_WRITE set at this point. */ if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { /* * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a * VM_PFNMAP VMA. FS DAX also wants ops->pfn_mkwrite called. * * We should not cow pages in a shared writeable mapping. * Just mark the pages writable and/or call ops->pfn_mkwrite. */ if (!vmf->page || is_fsdax_page(vmf->page)) { vmf->page = NULL; return wp_pfn_shared(vmf); } return wp_page_shared(vmf, folio); }
/* * Private mapping: create an exclusive anonymous page copy if reuse * is impossible. We might miss VM_WRITE for FOLL_FORCE handling. * * If we encounter a page that is marked exclusive, we must reuse * the page without further checks. */ if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { if (!PageAnonExclusive(vmf->page)) SetPageAnonExclusive(vmf->page); if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; } wp_page_reuse(vmf, folio); return 0; } /* * Ok, we need to copy. Oh, well.. */ if (folio) folio_get(folio);
pte_unmap_unlock(vmf->pte, vmf->ptl);#ifdef CONFIG_KSM if (folio && folio_test_ksm(folio)) count_vm_event(COW_KSM);#endif return wp_page_copy(vmf);}
static inline void unmap_mapping_range_tree(struct address_space *mapping, pgoff_t first_index, pgoff_t last_index, struct zap_details *details){ struct vm_area_struct *vma; struct mmu_gather tlb;
mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) { const pgoff_t start_idx = max(first_index, vma_start_pgoff(vma)); const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1; const pgoff_t offset = start_idx - vma_start_pgoff(vma); const unsigned long offset_bytes = offset << PAGE_SHIFT; const unsigned long start = vma->vm_start + offset_bytes; const unsigned long size = (end_idx - start_idx) << PAGE_SHIFT;
tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); }}
/** * unmap_mapping_folio() - Unmap single folio from processes. * @folio: The locked folio to be unmapped. * * Unmap this folio from any userspace process which still has it mmaped. * Typically, for efficiency, the range of nearby pages has already been * unmapped by unmap_mapping_pages() or unmap_mapping_range(). But once * truncation or invalidation holds the lock on a folio, it may find that * the page has been remapped again: and then uses unmap_mapping_folio() * to unmap it finally. */void unmap_mapping_folio(struct folio *folio){ struct address_space *mapping = folio->mapping; struct zap_details details = { }; pgoff_t first_index; pgoff_t last_index;
VM_BUG_ON(!folio_test_locked(folio));
first_index = folio->index; last_index = folio_next_index(folio) - 1;
details.skip_cows = true; details.single_folio = folio; details.zap_flags = ZAP_FLAG_DROP_MARKER;
i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping);}
/** * unmap_mapping_pages() - Unmap pages from processes. * @mapping: The address space containing pages to be unmapped. * @start: Index of first page to be unmapped. * @nr: Number of pages to be unmapped. 0 to unmap to end of file. * @even_cows: Whether to unmap even private COWed pages. * * Unmap the pages in this address space from any userspace process which * has them mmaped. Generally, you want to remove COWed pages as well when * a file is being truncated, but not when invalidating pages from the page * cache. */void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows){ struct zap_details details = { }; pgoff_t first_index = start; pgoff_t last_index = start + nr - 1;
details.skip_cows = !even_cows; if (last_index < first_index) last_index = ULONG_MAX;
i_mmap_lock_read(mapping); if (unlikely(mapping_mapped(mapping))) unmap_mapping_range_tree(mapping, first_index, last_index, &details); i_mmap_unlock_read(mapping);}EXPORT_SYMBOL_GPL(unmap_mapping_pages);
/** * unmap_mapping_range - unmap the portion of all mmaps in the specified * address_space corresponding to the specified byte range in the underlying * file. * * @mapping: the address space containing mmaps to be unmapped. * @holebegin: byte in first page to unmap, relative to the start of * the underlying file. This will be rounded down to a PAGE_SIZE * boundary. Note that this is different from truncate_pagecache(), which * must keep the partial page. In contrast, we must get rid of * partial pages. * @holelen: size of prospective hole in bytes. This will be rounded * up to a PAGE_SIZE boundary. A holelen of zero truncates to the * end of the file. * @even_cows: 1 when truncating a file, unmap even private COWed pages; * but 0 when invalidating pagecache, don't throw away private data. */void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows){ pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT; pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
/* Check for overflow. */ if (sizeof(holelen) > sizeof(hlen)) { long long holeend = (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; if (holeend & ~(long long)ULONG_MAX) hlen = ULONG_MAX - hba + 1; }
unmap_mapping_pages(mapping, hba, hlen, even_cows);}EXPORT_SYMBOL(unmap_mapping_range);
/* * Restore a potential device exclusive pte to a working pte entry */static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf){ struct folio *folio = page_folio(vmf->page); struct vm_area_struct *vma = vmf->vma; struct mmu_notifier_range range; vm_fault_t ret;
/* * We need a reference to lock the folio because we don't hold * the PTL so a racing thread can remove the device-exclusive * entry and unmap it. If the folio is free the entry must * have been removed already. If it happens to have already * been re-allocated after being freed all we do is lock and * unlock it. */ if (!folio_try_get(folio)) return 0;
ret = folio_lock_or_retry(folio, vmf); if (ret) { folio_put(folio); return ret; } mmu_notifier_range_init_owner(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, vmf->address & PAGE_MASK, (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL); mmu_notifier_invalidate_range_start(&range);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) restore_exclusive_pte(vma, folio, vmf->page, vmf->address, vmf->pte, vmf->orig_pte);
if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl); folio_unlock(folio); folio_put(folio);
mmu_notifier_invalidate_range_end(&range); return 0;}
/* * Check if we should call folio_free_swap to free the swap cache. * folio_free_swap only frees the swap cache to release the slot if swap * count is zero, so we don't need to check the swap count here. */static inline bool should_try_to_free_swap(struct swap_info_struct *si, struct folio *folio, struct vm_area_struct *vma, bool exclusive, unsigned int fault_flags){ if (!folio_test_swapcache(folio)) return false; /* * Always try to free swap cache for SWP_SYNCHRONOUS_IO devices. Swap * cache can help save some IO or memory overhead, but these devices * are fast, and meanwhile, swap cache pinning the slot deferring the * release of metadata or fragmentation is a more critical issue. */ if (data_race(si->flags & SWP_SYNCHRONOUS_IO)) return true; if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) || folio_test_mlocked(folio)) return true;
/* * Free the swapcache only if we are the exclusive user and * this is a write fault. */ return (fault_flags & FAULT_FLAG_WRITE) && exclusive;}
static vm_fault_t pte_marker_clear(struct vm_fault *vmf){ vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) return 0; /* * Be careful so that we will only recover a special uffd-wp pte into a * none pte. Otherwise it means the pte could have changed, so retry. * * This should also cover the case where e.g. the pte changed * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED. * So pte_is_marker() check is not enough to safely drop the pte. */ if (pte_same(vmf->orig_pte, ptep_get(vmf->pte))) pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); return 0;}
static vm_fault_t do_pte_missing(struct vm_fault *vmf){ if (vma_is_anonymous(vmf->vma)) return do_anonymous_page(vmf); else return do_fault(vmf);}
/* * This is actually a page-missing access, but with uffd-wp special pte * installed. It means this pte was wr-protected before being unmapped. */static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf){ /* * Just in case there're leftover special ptes even after the region * got unregistered - we can simply clear them. */ if (unlikely(!userfaultfd_wp(vmf->vma))) return pte_marker_clear(vmf);
return do_pte_missing(vmf);}
static vm_fault_t handle_pte_marker(struct vm_fault *vmf){ const softleaf_t entry = softleaf_from_pte(vmf->orig_pte); const pte_marker marker = softleaf_to_marker(entry);
/* * PTE markers should never be empty. If anything weird happened, * the best thing to do is to kill the process along with its mm. */ if (WARN_ON_ONCE(!marker)) return VM_FAULT_SIGBUS;
/* Higher priority than uffd-wp when data corrupted */ if (marker & PTE_MARKER_POISONED) return VM_FAULT_HWPOISON;
/* Hitting a guard page is always a fatal condition. */ if (marker & PTE_MARKER_GUARD) return VM_FAULT_SIGSEGV;
if (softleaf_is_uffd_wp_marker(entry)) return pte_marker_handle_uffd_wp(vmf);
/* This is an unknown pte marker */ return VM_FAULT_SIGBUS;}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE/* * Check if the PTEs within a range are contiguous swap entries. */static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages){ unsigned long addr; int idx; pte_t pte;
addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); idx = (vmf->address - addr) / PAGE_SIZE; pte = ptep_get(ptep);
if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx))) return false; /* * swap_read_folio() can't handle the case a large folio is hybridly * from different backends. And they are likely corner cases. Similar * things might be added once zswap support large folios. */ if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages) return false; return true;}
static inline unsigned long thp_swap_suitable_orders(pgoff_t swp_offset, unsigned long addr, unsigned long orders){ int order, nr;
order = highest_order(orders);
/* * To swap in a THP with nr pages, we require that its first swap_offset * is aligned with that number, as it was when the THP was swapped out. * This helps filter out most invalid entries. */ while (orders) { nr = 1 << order; if ((addr >> PAGE_SHIFT) % nr == swp_offset % nr) break; order = next_order(&orders, order); }
return orders;}
static unsigned long thp_swapin_suitable_orders(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; unsigned long orders; unsigned long addr; softleaf_t entry; spinlock_t *ptl; pte_t *pte; int order;
/* * If uffd is active for the vma we need per-page fault fidelity to * maintain the uffd semantics. */ if (unlikely(userfaultfd_armed(vma))) return 0;
/* * A large swapped out folio could be partially or fully in zswap. We * lack handling for such cases, so fallback to swapping in order-0 * folio. */ if (!zswap_never_enabled()) return 0;
entry = softleaf_from_pte(vmf->orig_pte); /* * Get a list of all the (large) orders below PMD_ORDER that are enabled * and suitable for swapping THP. */ orders = thp_vma_allowable_orders(vma, vma->vm_flags, TVA_PAGEFAULT, BIT(PMD_ORDER) - 1); orders = thp_vma_suitable_orders(vma, vmf->address, orders); orders = thp_swap_suitable_orders(swp_offset(entry), vmf->address, orders);
if (!orders) return 0;
pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address & PMD_MASK, &ptl); if (unlikely(!pte)) return 0;
/* * For do_swap_page, find the highest order where the aligned range is * completely swap entries with contiguous swap offsets. */ order = highest_order(orders); while (orders) { addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order); if (can_swapin_thp(vmf, pte + pte_index(addr), 1 << order)) break; order = next_order(&orders, order); }
pte_unmap_unlock(pte, ptl);
return orders;}#else /* !CONFIG_TRANSPARENT_HUGEPAGE */static unsigned long thp_swapin_suitable_orders(struct vm_fault *vmf){ return 0;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
/* Sanity check that a folio is fully exclusive */static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, unsigned int nr_pages){ /* Called under PT locked and folio locked, the swap count is stable */ do { VM_WARN_ON_ONCE_FOLIO(__swap_count(entry) != 1, folio); entry.val++; } while (--nr_pages);}
/* * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK), and pte mapped but not yet locked. * We return with pte unmapped and unlocked. * * When returning, the lock may have been released in the same cases * as done by filemap_fault(). */vm_fault_t do_swap_page(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct folio *swapcache = NULL, *folio; struct page *page; struct swap_info_struct *si = NULL; rmap_t rmap_flags = RMAP_NONE; bool exclusive = false; bool rwp_restore = false; softleaf_t entry; pte_t pte; vm_fault_t ret = 0; int nr_pages; unsigned long page_idx; unsigned long address; pte_t *ptep;
if (!pte_unmap_same(vmf)) goto out;
entry = softleaf_from_pte(vmf->orig_pte); if (unlikely(!softleaf_is_swap(entry))) { if (softleaf_is_migration(entry)) { migration_entry_wait(vma->vm_mm, vmf->pmd, vmf->address); } else if (softleaf_is_device_exclusive(entry)) { vmf->page = softleaf_to_page(entry); ret = remove_device_exclusive_entry(vmf); } else if (softleaf_is_device_private(entry)) { if (vmf->flags & FAULT_FLAG_VMA_LOCK) { /* * migrate_to_ram is not yet ready to operate * under VMA lock. */ vma_end_read(vma); ret = VM_FAULT_RETRY; goto out; }
vmf->page = softleaf_to_page(entry); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) goto unlock;
/* * Get a folio reference while we know the folio can't * be freed. */ folio = page_folio(vmf->page); if (folio_trylock(folio)) { struct dev_pagemap *pgmap;
folio_get(folio); pte_unmap_unlock(vmf->pte, vmf->ptl); pgmap = page_pgmap(vmf->page); ret = pgmap->ops->migrate_to_ram(vmf); /* migrate_to_ram() might have split the folio. */ folio = page_folio(vmf->page); folio_unlock(folio); folio_put(folio); } else { pte_unmap(vmf->pte); softleaf_entry_wait_on_locked(entry, vmf->ptl); } } else if (softleaf_is_hwpoison(entry)) { ret = VM_FAULT_HWPOISON; } else if (softleaf_is_marker(entry)) { ret = handle_pte_marker(vmf); } else { print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL); ret = VM_FAULT_SIGBUS; } goto out; }
/* Prevent swapoff from happening to us, and reject a bad entry. */ si = get_swap_device(entry); if (IS_ERR_OR_NULL(si)) { if (IS_ERR(si)) ret = VM_FAULT_SIGBUS; goto out; }
folio = swap_cache_get_folio(entry); if (folio) swap_update_readahead(folio, vma, vmf->address); if (!folio) { /* Swapin bypasses readahead for SWP_SYNCHRONOUS_IO devices */ if (data_race(si->flags & SWP_SYNCHRONOUS_IO)) folio = swapin_sync(entry, GFP_HIGHUSER_MOVABLE, thp_swapin_suitable_orders(vmf) | BIT(0), vmf, NULL, 0); else folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf);
if (IS_ERR_OR_NULL(folio)) { /* * Back out if somebody else faulted in this pte * while we released the pte lock. */ vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) ret = VM_FAULT_OOM; folio = NULL; goto unlock; }
/* Had to read the page from swap area: Major fault */ ret = VM_FAULT_MAJOR; count_vm_event(PGMAJFAULT); count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); }
swapcache = folio; ret |= folio_lock_or_retry(folio, vmf); if (ret & VM_FAULT_RETRY) goto out_release;
page = folio_file_page(folio, swp_offset(entry)); /* * Make sure folio_free_swap() or swapoff did not release the * swapcache from under us. The page pin, and pte_same test * below, are not enough to exclude that. Even if it is still * swapcache, we need to check that the page's swap has not * changed. */ if (unlikely(!folio_matches_swap_entry(folio, entry))) goto out_page;
if (unlikely(PageHWPoison(page))) { /* * hwpoisoned dirty swapcache pages are kept for killing * owner processes (which may be unknown at hwpoison time) */ ret = VM_FAULT_HWPOISON; goto out_page; }
/* * KSM sometimes has to copy on read faults, for example, if * folio->index of non-ksm folios would be nonlinear inside the * anon VMA -- the ksm flag is lost on actual swapout. */ folio = ksm_might_need_to_copy(folio, vma, vmf->address); if (unlikely(!folio)) { ret = VM_FAULT_OOM; folio = swapcache; goto out_page; } else if (unlikely(folio == ERR_PTR(-EHWPOISON))) { ret = VM_FAULT_HWPOISON; folio = swapcache; goto out_page; } else if (folio != swapcache) page = folio_page(folio, 0);
folio_throttle_swaprate(folio, GFP_KERNEL);
/* * Back out if somebody else already faulted in this pte. */ vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) goto out_nomap;
if (unlikely(!folio_test_uptodate(folio))) { ret = VM_FAULT_SIGBUS; goto out_nomap; }
nr_pages = 1; page_idx = 0; address = vmf->address; ptep = vmf->pte; if (folio_test_large(folio) && folio_test_swapcache(folio)) { int nr = folio_nr_pages(folio); unsigned long idx = folio_page_idx(folio, page); unsigned long folio_start = address - idx * PAGE_SIZE; unsigned long folio_end = folio_start + nr * PAGE_SIZE; pte_t *folio_ptep; pte_t folio_pte;
if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start))) goto check_folio; if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end))) goto check_folio;
folio_ptep = vmf->pte - idx; folio_pte = ptep_get(folio_ptep); if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) || swap_pte_batch(folio_ptep, nr, folio_pte) != nr) goto check_folio;
page_idx = idx; address = folio_start; ptep = folio_ptep; nr_pages = nr; entry = folio->swap; page = &folio->page; }
check_folio: /* * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte * must never point at an anonymous page in the swapcache that is * PG_anon_exclusive. Sanity check that this holds and especially, that * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity * check after taking the PT lock and making sure that nobody * concurrently faulted in this page and set PG_anon_exclusive. */ BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio)); BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
/* * If a large folio already belongs to anon mapping, then we * can just go on and map it partially. * If not, with the large swapin check above failing, the page table * have changed, so sub pages might got charged to the wrong cgroup, * or even should be shmem. So we have to free it and fallback. * Nothing should have touched it, both anon and shmem checks if a * large folio is fully applicable before use. * * This will be removed once we unify folio allocation in the swap cache * layer, where allocation of a folio stabilizes the swap entries. */ if (!folio_test_anon(folio) && folio_test_large(folio) && nr_pages != folio_nr_pages(folio)) { if (!WARN_ON_ONCE(folio_test_dirty(folio))) swap_cache_del_folio(folio); goto out_nomap; }
/* * Check under PT lock (to protect against concurrent fork() sharing * the swap entry concurrently) for certainly exclusive pages. */ if (!folio_test_ksm(folio)) { /* * The can_swapin_thp check above ensures all PTE have * same exclusiveness. Checking just one PTE is fine. */ exclusive = pte_swp_exclusive(vmf->orig_pte); if (exclusive) check_swap_exclusive(folio, entry, nr_pages); if (folio != swapcache) { /* * We have a fresh page that is not exposed to the * swapcache -> certainly exclusive. */ exclusive = true; } else if (exclusive && folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES)) { /* * This is tricky: not all swap backends support * concurrent page modifications while under writeback. * * So if we stumble over such a page in the swapcache * we must not set the page exclusive, otherwise we can * map it writable without further checks and modify it * while still under writeback. * * For these problematic swap backends, simply drop the * exclusive marker: this is perfectly fine as we start * writeback only if we fully unmapped the page and * there are no unexpected references on the page after * unmapping succeeded. After fully unmapped, no * further GUP references (FOLL_GET and FOLL_PIN) can * appear, so dropping the exclusive marker and mapping * it only R/O is fine. */ exclusive = false; } }
/* * Some architectures may have to restore extra metadata to the page * when reading from swap. This metadata may be indexed by swap entry * so this must be called before folio_put_swap(). */ arch_swap_restore(folio_swap(entry, folio), folio);
add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages); add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages); pte = mk_pte(page, vma->vm_page_prot); if (pte_swp_soft_dirty(vmf->orig_pte)) pte = pte_mksoft_dirty(pte); if (pte_swp_uffd(vmf->orig_pte)) pte = pte_mkuffd(pte);
/* * A page reclaimed while RWP-protected carries the uffd bit on * its swap entry. Re-apply PAGE_NONE on swap-in so the first access * still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD. */ if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) { pte = pte_modify(pte, PAGE_NONE); rwp_restore = true; }
/* * Similar logic as in do_wp_page(); however, optimize for pages that * are certainly exclusive. * * Skip the write upgrade for an RWP-restored pte: it must stay * PROT_NONE so the access retries through the RWP fault path * (do_uffd_rwp()) rather than being made writable here. */ if (exclusive) { if (!rwp_restore && (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && !pte_needs_soft_dirty_wp(vma, pte)) { pte = pte_mkwrite(pte, vma); if (vmf->flags & FAULT_FLAG_WRITE) pte = pte_mkdirty(pte); } rmap_flags |= RMAP_EXCLUSIVE; } folio_ref_add(folio, nr_pages - 1); flush_icache_pages(vma, page, nr_pages); vmf->orig_pte = pte_advance_pfn(pte, page_idx);
/* ksm created a completely new copy */ if (unlikely(folio != swapcache)) { folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); folio_put_swap(swapcache, NULL); } else if (!folio_test_anon(folio)) { /* * We currently only expect !anon folios that are fully * mappable. See the comment after can_swapin_thp above. */ VM_WARN_ON_ONCE_FOLIO(folio_nr_pages(folio) != nr_pages, folio); VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); folio_add_new_anon_rmap(folio, vma, address, rmap_flags); folio_put_swap(folio, NULL); } else { VM_WARN_ON_ONCE(nr_pages != 1 && nr_pages != folio_nr_pages(folio)); folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address, rmap_flags); folio_put_swap(folio, nr_pages == 1 ? page : NULL); }
VM_BUG_ON(!folio_test_anon(folio) || (pte_write(pte) && !PageAnonExclusive(page))); set_ptes(vma->vm_mm, address, ptep, pte, nr_pages); arch_do_swap_page_nr(vma->vm_mm, vma, address, pte, pte, nr_pages);
/* * Remove the swap entry and conditionally try to free up the swapcache. * Do it after mapping, so raced page faults will likely see the folio * in swap cache and wait on the folio lock. */ if (should_try_to_free_swap(si, folio, vma, exclusive, vmf->flags)) folio_free_swap(folio);
folio_unlock(folio); if (unlikely(folio != swapcache)) { /* * Hold the lock to avoid the swap entry to be reused * until we take the PT lock for the pte_same() check * (to avoid false positives from pte_same). For * further safety release the lock after the folio_put_swap * so that the swap count won't change under a * parallel locked swapcache. */ folio_unlock(swapcache); folio_put(swapcache); }
/* * For an RWP-restored pte, leave it PROT_NONE and let the write * retry through the RWP fault path; do not COW it here, which would * drop the marker for a non-exclusive page. */ if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) { ret |= do_wp_page(vmf); if (ret & VM_FAULT_ERROR) ret &= VM_FAULT_ERROR; goto out; }
/* No need to invalidate - it was non-present before */ update_mmu_cache_range(vmf, vma, address, ptep, nr_pages);unlock: if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl);out: if (!IS_ERR_OR_NULL(si)) put_swap_device(si); return ret;out_nomap: if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl);out_page: if (folio_test_swapcache(folio)) folio_free_swap(folio); folio_unlock(folio);out_release: folio_put(folio); if (folio != swapcache) { folio_unlock(swapcache); folio_put(swapcache); } if (si) put_swap_device(si); return ret;}
static bool pte_range_none(pte_t *pte, int nr_pages){ int i;
for (i = 0; i < nr_pages; i++) { if (!pte_none(ptep_get_lockless(pte + i))) return false; }
return true;}
static struct folio *alloc_anon_folio(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma;#ifdef CONFIG_TRANSPARENT_HUGEPAGE unsigned long orders; struct folio *folio; unsigned long addr; pte_t *pte; gfp_t gfp; int order;
/* * If uffd is active for the vma we need per-page fault fidelity to * maintain the uffd semantics. */ if (unlikely(userfaultfd_armed(vma))) goto fallback;
/* * Get a list of all the (large) orders below PMD_ORDER that are enabled * for this vma. Then filter out the orders that can't be allocated over * the faulting address and still be fully contained in the vma. */ orders = thp_vma_allowable_orders(vma, vma->vm_flags, TVA_PAGEFAULT, BIT(PMD_ORDER) - 1); orders = thp_vma_suitable_orders(vma, vmf->address, orders);
if (!orders) goto fallback;
pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK); if (!pte) return ERR_PTR(-EAGAIN);
/* * Find the highest order where the aligned range is completely * pte_none(). Note that all remaining orders will be completely * pte_none(). */ order = highest_order(orders); while (orders) { addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order); if (pte_range_none(pte + pte_index(addr), 1 << order)) break; order = next_order(&orders, order); }
pte_unmap(pte);
if (!orders) goto fallback;
/* Try allocating the highest of the remaining orders. */ gfp = vma_thp_gfp_mask(vma); while (orders) { addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order); folio = vma_alloc_folio(gfp, order, vma, addr); if (!folio) goto next; if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); folio_put(folio); goto next; } if (order > 1 && folio_memcg_alloc_deferred(folio)) { folio_put(folio); goto fallback; } folio_throttle_swaprate(folio, gfp); /* * When a folio is not zeroed during allocation * (__GFP_ZERO not used) or user folios require special * handling, folio_zero_user() is used to make sure * that the page corresponding to the faulting address * will be hot in the cache after zeroing. */ if (user_alloc_needs_zeroing()) folio_zero_user(folio, vmf->address); return folio;next: count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); order = next_order(&orders, order); }
fallback:#endif return folio_prealloc(vma->vm_mm, vma, vmf->address, true);}
void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, struct vm_area_struct *vma, unsigned long addr, bool uffd_wp){ const unsigned int nr_pages = folio_nr_pages(folio); pte_t entry = folio_mk_pte(folio, vma->vm_page_prot);
entry = pte_sw_mkyoung(entry);
if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); if (uffd_wp) entry = pte_mkuffd(entry);
folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); update_mmu_cache_range(NULL, vma, addr, pte, nr_pages);}
static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, struct vm_area_struct *vma, unsigned long addr, bool uffd_wp){ const unsigned int order = folio_order(folio);
map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order); count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC);}
/* * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK), and pte unmapped and unlocked. * We return with the lock still held, but pte unmapped and unlocked. * If VM_FAULT_RETRY is returned, the lock may have been released. */static vm_fault_t do_anonymous_page(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; unsigned long addr = vmf->address; struct folio *folio; vm_fault_t ret = 0; int nr_pages; pte_t entry;
/* File mapping without ->vm_ops ? */ if (vma->vm_flags & VM_SHARED) return VM_FAULT_SIGBUS;
/* * Use pte_alloc() instead of pte_alloc_map(), so that OOM can * be distinguished from a transient failure of pte_offset_map(). */ if (pte_alloc(vma->vm_mm, vmf->pmd)) return VM_FAULT_OOM;
/* Use the zero-page for reads */ if (!(vmf->flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(vma->vm_mm)) { entry = pte_mkspecial(pfn_pte(zero_pfn(vmf->address), vma->vm_page_prot)); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock; if (vmf_pte_changed(vmf)) { update_mmu_tlb(vma, vmf->address, vmf->pte); goto unlock; } ret = check_stable_address_space(vma->vm_mm); if (ret) goto unlock; /* Deliver the page fault to userland, check inside PT lock */ if (userfaultfd_missing(vma)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return handle_userfault(vmf, VM_UFFD_MISSING); } if (vmf_orig_pte_uffd_wp(vmf)) entry = pte_mkuffd(entry); set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
/* No need to invalidate - it was non-present before */ update_mmu_cache(vma, addr, vmf->pte); goto unlock; }
/* Allocate our own private page. */ ret = vmf_anon_prepare(vmf); if (ret) return ret; /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) return 0; if (!folio) goto oom;
nr_pages = folio_nr_pages(folio); addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
/* * The memory barrier inside __folio_mark_uptodate makes sure that * preceding stores to the page contents become visible before * the set_pte_at() write. */ __folio_mark_uptodate(folio);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); if (!vmf->pte) goto release; if (nr_pages == 1 && vmf_pte_changed(vmf)) { update_mmu_tlb(vma, addr, vmf->pte); goto release; } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) { update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages); goto release; }
ret = check_stable_address_space(vma->vm_mm); if (ret) goto release;
/* Deliver the page fault to userland, check inside PT lock */ if (userfaultfd_missing(vma)) { pte_unmap_unlock(vmf->pte, vmf->ptl); folio_put(folio); return handle_userfault(vmf, VM_UFFD_MISSING); } map_anon_folio_pte_pf(folio, vmf->pte, vma, addr, vmf_orig_pte_uffd_wp(vmf));unlock: if (vmf->pte) pte_unmap_unlock(vmf->pte, vmf->ptl); return ret;release: folio_put(folio); goto unlock;oom: return VM_FAULT_OOM;}
/* * Either the VMA lock or the mmap_lock must have been held on entry * (see FAULT_FLAG_VMA_LOCK) and may have been released depending on * flags and vma->vm_ops->fault() return value. * See filemap_fault() and __folio_lock_or_retry(). */static vm_fault_t __do_fault(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct folio *folio; vm_fault_t ret;
/* * Preallocate pte before we take folio lock because this might lead to * deadlocks for memcg reclaim which waits for folios under writeback: * folio_lock(A) * folio_set_writeback(A) * folio_unlock(A) * folio_lock(B) * folio_lock(B) * pte_alloc_one * shrink_folio_list * folio_wait_writeback(A) * folio_set_writeback(B) * folio_unlock(B) * # flush A, B to clear the writeback */ if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) { vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; }
ret = vma->vm_ops->fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY | VM_FAULT_DONE_COW))) return ret;
folio = page_folio(vmf->page); if (unlikely(PageHWPoison(vmf->page))) { vm_fault_t poisonret = VM_FAULT_HWPOISON; if (ret & VM_FAULT_LOCKED) { if (folio_mapped(folio)) unmap_mapping_folio(folio); /* Retry if a clean folio was removed from the cache. */ if (mapping_evict_folio(folio->mapping, folio)) poisonret = VM_FAULT_NOPAGE; folio_unlock(folio); } folio_put(folio); vmf->page = NULL; return poisonret; }
if (unlikely(!(ret & VM_FAULT_LOCKED))) folio_lock(folio); else VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);
return ret;}
#ifdef CONFIG_TRANSPARENT_HUGEPAGEstatic void deposit_prealloc_pte(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma;
pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); /* * We are going to consume the prealloc table, * count that as nr_ptes. */ mm_inc_nr_ptes(vma->vm_mm); vmf->prealloc_pte = NULL;}
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page){ struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; unsigned long haddr = vmf->address & HPAGE_PMD_MASK; pmd_t entry; vm_fault_t ret = VM_FAULT_FALLBACK;
/* * It is too late to allocate a small folio, we already have a large * folio in the pagecache: especially s390 KVM cannot tolerate any * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any * PMD mappings if THPs are disabled. As we already have a THP, * behave as if we are forcing a collapse. */ if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags, /* forced_collapse=*/ true)) return ret;
if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) return ret;
if (!is_pmd_order(folio_order(folio))) return ret; page = &folio->page;
/* * Just backoff if any subpage of a THP is corrupted otherwise * the corrupted page may mapped by PMD silently to escape the * check. This kind of THP just can be PTE mapped. Access to * the corrupted subpage should trigger SIGBUS as expected. */ if (unlikely(folio_test_has_hwpoisoned(folio))) return ret;
/* * Archs like ppc64 need additional space to store information * related to pte entry. Use the preallocated table for that. */ if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) { vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; }
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); if (unlikely(!pmd_none(*vmf->pmd))) goto out;
flush_icache_pages(vma, page, HPAGE_PMD_NR);
entry = folio_mk_pmd(folio, vma->vm_page_prot); if (write) entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); folio_add_file_rmap_pmd(folio, page, vma);
/* * deposit and withdraw with pmd lock held */ if (arch_needs_pgtable_deposit()) deposit_prealloc_pte(vmf);
set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
update_mmu_cache_pmd(vma, haddr, vmf->pmd);
/* fault is handled */ ret = 0; count_vm_event(THP_FILE_MAPPED);out: spin_unlock(vmf->ptl); return ret;}#elsevm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page){ return VM_FAULT_FALLBACK;}#endif
/** * set_pte_range - Set a range of PTEs to point to pages in a folio. * @vmf: Fault description. * @folio: The folio that contains @page. * @page: The first page to create a PTE for. * @nr: The number of PTEs to create. * @addr: The first address to create a PTE for. */void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr){ struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE); pte_t entry;
flush_icache_pages(vma, page, nr); entry = mk_pte(page, vma->vm_page_prot);
if (prefault && arch_wants_old_prefaulted_pte()) entry = pte_mkold(entry); else entry = pte_sw_mkyoung(entry);
if (write) entry = maybe_mkwrite(pte_mkdirty(entry), vma); else if (pte_write(entry) && folio_test_dirty(folio)) entry = pte_mkdirty(entry); if (unlikely(vmf_orig_pte_uffd_wp(vmf))) entry = pte_mkuffd(entry); /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { VM_BUG_ON_FOLIO(nr != 1, folio); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); } else { folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
/* no need to invalidate: a not-present page won't be cached */ update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);}
static bool vmf_pte_changed(struct vm_fault *vmf){ if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID) return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
return !pte_none(ptep_get(vmf->pte));}
/** * finish_fault - finish page fault once we have prepared the page to fault * * @vmf: structure describing the fault * * This function handles all that is needed to finish a page fault once the * page to fault in is prepared. It handles locking of PTEs, inserts PTE for * given page, adds reverse page mapping, handles memcg charges and LRU * addition. * * The function expects the page to be locked and on success it consumes a * reference of a page being mapped (for the PTE which maps it). * * Return: %0 on success, %VM_FAULT_ code in case of error. */vm_fault_t finish_fault(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct page *page; struct folio *folio; vm_fault_t ret; bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); int type, nr_pages; unsigned long addr; bool needs_fallback = false;
fallback: addr = vmf->address;
/* Did we COW the page? */ if (is_cow) page = vmf->cow_page; else page = vmf->page;
folio = page_folio(page); /* * check even for read faults because we might have lost our CoWed * page */ if (!(vma->vm_flags & VM_SHARED)) { ret = check_stable_address_space(vma->vm_mm); if (ret) return ret; }
if (!needs_fallback && vma->vm_file) { struct address_space *mapping = vma->vm_file->f_mapping; pgoff_t file_end;
file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
/* * Do not allow to map with PTEs beyond i_size and with PMD * across i_size to preserve SIGBUS semantics. * * Make an exception for shmem/tmpfs that for long time * intentionally mapped with PMDs across i_size. */ needs_fallback = !shmem_mapping(mapping) && file_end < folio_next_index(folio); }
if (pmd_none(*vmf->pmd)) { if (!needs_fallback && folio_test_pmd_mappable(folio)) { ret = do_set_pmd(vmf, folio, page); if (ret != VM_FAULT_FALLBACK) return ret; }
if (vmf->prealloc_pte) pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) return VM_FAULT_OOM; }
nr_pages = folio_nr_pages(folio);
/* Using per-page fault to maintain the uffd semantics */ if (unlikely(userfaultfd_armed(vma)) || unlikely(needs_fallback)) { nr_pages = 1; } else if (nr_pages > 1) { pgoff_t idx = folio_page_idx(folio, page); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); /* The index of the entry in the pagetable for fault page. */ pgoff_t pte_off = pte_index(vmf->address);
/* * Fallback to per-page fault in case the folio size in page * cache beyond the VMA limits and PMD pagetable limits. */ if (unlikely(vma_off < idx || vma_off + (nr_pages - idx) > vma_pages(vma) || pte_off < idx || pte_off + (nr_pages - idx) > PTRS_PER_PTE)) { nr_pages = 1; } else { /* Now we can set mappings for the whole large folio. */ addr = vmf->address - idx * PAGE_SIZE; page = &folio->page; } }
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); if (!vmf->pte) return VM_FAULT_NOPAGE;
/* Re-check under ptl */ if (nr_pages == 1 && unlikely(vmf_pte_changed(vmf))) { update_mmu_tlb(vma, addr, vmf->pte); ret = VM_FAULT_NOPAGE; goto unlock; } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) { needs_fallback = true; pte_unmap_unlock(vmf->pte, vmf->ptl); goto fallback; }
folio_ref_add(folio, nr_pages - 1); set_pte_range(vmf, folio, page, nr_pages, addr); type = is_cow ? MM_ANONPAGES : mm_counter_file(folio); add_mm_counter(vma->vm_mm, type, nr_pages); ret = 0;
unlock: pte_unmap_unlock(vmf->pte, vmf->ptl); return ret;}
static unsigned long fault_around_pages __read_mostly = 65536 >> PAGE_SHIFT;
#ifdef CONFIG_DEBUG_FSstatic int fault_around_bytes_get(void *data, u64 *val){ *val = fault_around_pages << PAGE_SHIFT; return 0;}
/* * fault_around_bytes must be rounded down to the nearest page order as it's * what do_fault_around() expects to see. */static int fault_around_bytes_set(void *data, u64 val){ if (val / PAGE_SIZE > PTRS_PER_PTE) return -EINVAL;
/* * The minimum value is 1 page, however this results in no fault-around * at all. See should_fault_around(). */ val = max(val, PAGE_SIZE); fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT;
return 0;}DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops, fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
static int __init fault_around_debugfs(void){ debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL, &fault_around_bytes_fops); return 0;}late_initcall(fault_around_debugfs);#endif
/* * do_fault_around() tries to map few pages around the fault address. The hope * is that the pages will be needed soon and this will lower the number of * faults to handle. * * It uses vm_ops->map_pages() to map the pages, which skips the page if it's * not ready to be mapped: not up-to-date, locked, etc. * * This function doesn't cross VMA or page table boundaries, in order to call * map_pages() and acquire a PTE lock only once. * * fault_around_pages defines how many pages we'll try to map. * do_fault_around() expects it to be set to a power of two less than or equal * to PTRS_PER_PTE. * * The virtual address of the area that we map is naturally aligned to * fault_around_pages * PAGE_SIZE rounded down to the machine page size * (and therefore to page order). This way it's easier to guarantee * that we don't cross page table boundaries. */static vm_fault_t do_fault_around(struct vm_fault *vmf){ pgoff_t nr_pages = READ_ONCE(fault_around_pages); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma); pgoff_t from_pte, to_pte; vm_fault_t ret;
/* The PTE offset of the start address, clamped to the VMA. */ from_pte = max(ALIGN_DOWN(pte_off, nr_pages), pte_off - min(pte_off, vma_off));
/* The PTE offset of the end address, clamped to the VMA and PTE. */ to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE, pte_off + vma_pages(vmf->vma) - vma_off) - 1;
if (pmd_none(*vmf->pmd)) { vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; }
rcu_read_lock(); ret = vmf->vma->vm_ops->map_pages(vmf, vmf->pgoff + from_pte - pte_off, vmf->pgoff + to_pte - pte_off); rcu_read_unlock();
return ret;}
/* Return true if we should do read fault-around, false otherwise */static inline bool should_fault_around(struct vm_fault *vmf){ /* No ->map_pages? No way to fault around... */ if (!vmf->vma->vm_ops->map_pages) return false;
if (uffd_disable_fault_around(vmf->vma)) return false;
/* A single page implies no faulting 'around' at all. */ return fault_around_pages > 1;}
static vm_fault_t do_read_fault(struct vm_fault *vmf){ vm_fault_t ret = 0; struct folio *folio;
/* * Let's call ->map_pages() first and use ->fault() as fallback * if page by the offset is not ready to be mapped (cold cache or * something). */ if (should_fault_around(vmf)) { ret = do_fault_around(vmf); if (ret) return ret; }
ret = vmf_can_call_fault(vmf); if (ret) return ret;
ret = __do_fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) return ret;
ret |= finish_fault(vmf); folio = page_folio(vmf->page); folio_unlock(folio); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) folio_put(folio); return ret;}
static vm_fault_t do_cow_fault(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct folio *folio; vm_fault_t ret;
ret = vmf_can_call_fault(vmf); if (!ret) ret = vmf_anon_prepare(vmf); if (ret) return ret;
folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); if (!folio) return VM_FAULT_OOM;
vmf->cow_page = &folio->page;
ret = __do_fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) goto uncharge_out; if (ret & VM_FAULT_DONE_COW) return ret;
if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { ret = VM_FAULT_HWPOISON; goto unlock; } __folio_mark_uptodate(folio);
ret |= finish_fault(vmf);unlock: unlock_page(vmf->page); put_page(vmf->page); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) goto uncharge_out; return ret;uncharge_out: folio_put(folio); return ret;}
static vm_fault_t do_shared_fault(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; vm_fault_t ret, tmp; struct folio *folio;
ret = vmf_can_call_fault(vmf); if (ret) return ret;
ret = __do_fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) return ret;
folio = page_folio(vmf->page);
/* * Check if the backing address space wants to know that the page is * about to become writable */ if (vma->vm_ops->page_mkwrite) { folio_unlock(folio); tmp = do_page_mkwrite(vmf, folio); if (unlikely(!tmp || (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { folio_put(folio); return tmp; } }
ret |= finish_fault(vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) { folio_unlock(folio); folio_put(folio); return ret; }
ret |= fault_dirty_shared_page(vmf); return ret;}
/* * We enter with either the VMA lock or the mmap_lock held (see * FAULT_FLAG_VMA_LOCK). * The lock may have been released depending on flags and our * return value. See filemap_fault() and __folio_lock_or_retry(). * If the lock is released, vma may become invalid (for example * by other thread calling munmap()). */static vm_fault_t do_fault(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct mm_struct *vm_mm = vma->vm_mm; vm_fault_t ret;
/* * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */ if (!vma->vm_ops->fault) { vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (unlikely(!vmf->pte)) ret = VM_FAULT_SIGBUS; else { /* * Make sure this is not a temporary clearing of pte * by holding ptl and checking again. A R/M/W update * of pte involves: take ptl, clearing the pte so that * we don't have concurrent modification by hardware * followed by an update. */ if (unlikely(pte_none(ptep_get(vmf->pte)))) ret = VM_FAULT_SIGBUS; else ret = VM_FAULT_NOPAGE;
pte_unmap_unlock(vmf->pte, vmf->ptl); } } else if (!(vmf->flags & FAULT_FLAG_WRITE)) ret = do_read_fault(vmf); else if (!(vma->vm_flags & VM_SHARED)) ret = do_cow_fault(vmf); else ret = do_shared_fault(vmf);
/* preallocated pagetable is unused: free it */ if (vmf->prealloc_pte) { pte_free(vm_mm, vmf->prealloc_pte); vmf->prealloc_pte = NULL; } return ret;}
int numa_migrate_check(struct folio *folio, struct vm_fault *vmf, unsigned long addr, int *flags, bool writable, int *last_cpupid){ struct vm_area_struct *vma = vmf->vma;
/* * Avoid grouping on RO pages in general. RO pages shouldn't hurt as * much anyway since they can be in shared cache state. This misses * the case where a mapping is writable but the process never writes * to it but pte_write gets cleared during protection updates and * pte_dirty has unpredictable behaviour between PTE scan updates, * background writeback, dirty balancing and application behaviour. */ if (!writable) *flags |= TNF_NO_GROUP;
/* * Flag if the folio is shared between multiple address spaces. This * is later used when determining whether to group tasks together */ if (folio_maybe_mapped_shared(folio) && (vma->vm_flags & VM_SHARED)) *flags |= TNF_SHARED; /* * For memory tiering mode, cpupid of slow memory page is used * to record page access time. So use default value. */ if (folio_use_access_time(folio)) *last_cpupid = (-1 & LAST_CPUPID_MASK); else *last_cpupid = folio_last_cpupid(folio);
/* Record the current PID accessing VMA */ vma_set_access_pid_bit(vma);
count_vm_numa_event(NUMA_HINT_FAULTS);#ifdef CONFIG_NUMA_BALANCING count_memcg_folio_events(folio, NUMA_HINT_FAULTS, 1);#endif if (folio_nid(folio) == numa_node_id()) { count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); *flags |= TNF_FAULT_LOCAL; }
return mpol_misplaced(folio, vmf, addr);}
static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long fault_addr, pte_t *fault_pte, bool writable){ pte_t pte, old_pte;
old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte); pte = pte_modify(old_pte, vma->vm_page_prot); pte = pte_mkyoung(pte); if (writable) pte = pte_mkwrite(pte, vma); ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte); update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1);}
static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade){ int nr = pte_pfn(fault_pte) - folio_pfn(folio); unsigned long start, end, addr = vmf->address; unsigned long addr_start = addr - (nr << PAGE_SHIFT); unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); pte_t *start_ptep;
/* Stay within the VMA and within the page table. */ start = max3(addr_start, pt_start, vma->vm_start); end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, vma->vm_end); start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT);
/* Restore all PTEs' mapping of the large folio */ for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { pte_t ptent = ptep_get(start_ptep); bool writable = false;
if (!pte_present(ptent) || !pte_protnone(ptent)) continue;
/* * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a * marker. Leave them alone -- rewriting to vm_page_prot would * stop the RWP trap. Gate on userfaultfd_rwp(vma) too: * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs * when applying PROT_NONE, and those still need rebuilding. */ if (userfaultfd_rwp(vma) && pte_uffd(ptent)) continue;
if (pfn_folio(pte_pfn(ptent)) != folio) continue;
if (!ignore_writable) { ptent = pte_modify(ptent, vma->vm_page_prot); writable = pte_write(ptent); if (!writable && pte_write_upgrade && can_change_pte_writable(vma, addr, ptent)) writable = true; }
numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable); }}
static vm_fault_t do_uffd_rwp(struct vm_fault *vmf){ pte_t pte;
if (!userfaultfd_rwp_async(vmf->vma)) { /* Sync mode: unmap PTE and deliver to userfaultfd handler */ pte_unmap(vmf->pte); return handle_userfault(vmf, VM_UFFD_RWP); }
spin_lock(vmf->ptl); if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; } pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot); /* pte_modify() preserves _PAGE_UFFD; drop it on resolution */ pte = pte_clear_uffd(pte); pte = pte_mkyoung(pte); if (!pte_write(pte) && vma_wants_manual_pte_write_upgrade(vmf->vma) && can_change_pte_writable(vmf->vma, vmf->address, pte)) pte = pte_mkwrite(pte, vmf->vma); set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte); update_mmu_cache(vmf->vma, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); return 0;}
static vm_fault_t do_numa_page(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; struct folio *folio = NULL; int nid = NUMA_NO_NODE; bool writable = false, ignore_writable = false; bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma); int last_cpupid; int target_nid; pte_t pte, old_pte; int flags = 0, nr_pages;
/* * The pte cannot be used safely until we verify, while holding the page * table lock, that its contents have not changed during fault handling. */ spin_lock(vmf->ptl); /* Read the live PTE from the page tables: */ old_pte = ptep_get(vmf->pte);
if (unlikely(!pte_same(old_pte, vmf->orig_pte))) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; }
pte = pte_modify(old_pte, vma->vm_page_prot);
/* * Detect now whether the PTE could be writable; this information * is only valid while holding the PT lock. */ writable = pte_write(pte); if (!writable && pte_write_upgrade && can_change_pte_writable(vma, vmf->address, pte)) writable = true;
folio = vm_normal_folio(vma, vmf->address, pte); if (!folio || folio_is_zone_device(folio)) goto out_map;
nid = folio_nid(folio); nr_pages = folio_nr_pages(folio);
target_nid = numa_migrate_check(folio, vmf, vmf->address, &flags, writable, &last_cpupid); if (target_nid == NUMA_NO_NODE) goto out_map; if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { flags |= TNF_MIGRATE_FAIL; goto out_map; } /* The folio is isolated and isolation code holds a folio reference. */ pte_unmap_unlock(vmf->pte, vmf->ptl); writable = false; ignore_writable = true;
/* Migrate to the requested node */ if (!migrate_misplaced_folio(folio, target_nid)) { nid = target_nid; flags |= TNF_MIGRATED; task_numa_fault(last_cpupid, nid, nr_pages, flags); return 0; }
flags |= TNF_MIGRATE_FAIL; vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (unlikely(!vmf->pte)) return 0; if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; }out_map: /* * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ if (folio && folio_test_large(folio)) numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, pte_write_upgrade); else numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, writable); pte_unmap_unlock(vmf->pte, vmf->ptl);
if (nid != NUMA_NO_NODE) task_numa_fault(last_cpupid, nid, nr_pages, flags); return 0;}
static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; if (vma_is_anonymous(vma)) return do_huge_pmd_anonymous_page(vmf); if (vma->vm_ops->huge_fault) return vma->vm_ops->huge_fault(vmf, PMD_ORDER); return VM_FAULT_FALLBACK;}
/* `inline' is required to avoid gcc 4.1.2 build error */static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf){ struct vm_area_struct *vma = vmf->vma; const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; vm_fault_t ret;
if (vma_is_anonymous(vma)) { if (likely(!unshare) && userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) { if (userfaultfd_wp_async(vmf->vma)) goto split; return handle_userfault(vmf, VM_UFFD_WP); } return do_huge_pmd_wp_page(vmf); }
if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { if (vma->vm_ops->huge_fault) { ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER); if (!(ret & VM_FAULT_FALLBACK)) return ret; } }
split: /* COW or write-notify handled on pte level: split pmd. */ __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
return VM_FAULT_FALLBACK;}
static vm_fault_t create_huge_pud(struct vm_fault *vmf){#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) struct vm_area_struct *vma = vmf->vma; /* No support for anonymous transparent PUD pages yet */ if (vma_is_anonymous(vma)) return VM_FAULT_FALLBACK; if (vma->vm_ops->huge_fault) return vma->vm_ops->huge_fault(vmf, PUD_ORDER);#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ return VM_FAULT_FALLBACK;}
static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud){#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) struct vm_area_struct *vma = vmf->vma; vm_fault_t ret;
/* No support for anonymous transparent PUD pages yet */ if (vma_is_anonymous(vma)) goto split; if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { if (vma->vm_ops->huge_fault) { ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER); if (!(ret & VM_FAULT_FALLBACK)) return ret; } }split: /* COW or write-notify not handled on PUD level: split pud.*/ __split_huge_pud(vma, vmf->pud, vmf->address);#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ return VM_FAULT_FALLBACK;}
/* * The page faults may be spurious because of the racy access to the * page table. For example, a non-populated virtual page is accessed * on 2 CPUs simultaneously, thus the page faults are triggered on * both CPUs. However, it's possible that one CPU (say CPU A) cannot * find the reason for the page fault if the other CPU (say CPU B) has * changed the page table before the PTE is checked on CPU A. Most of * the time, the spurious page faults can be ignored safely. However, * if the page fault is for the write access, it's possible that a * stale read-only TLB entry exists in the local CPU and needs to be * flushed on some architectures. This is called the spurious page * fault fixing. * * Note: flush_tlb_fix_spurious_fault() is defined as flush_tlb_page() * by default and used as such on most architectures, while * flush_tlb_fix_spurious_fault_pmd() is defined as NOP by default and * used as such on most architectures. */static void fix_spurious_fault(struct vm_fault *vmf, enum pgtable_level ptlevel){ /* Skip spurious TLB flush for retried page fault */ if (vmf->flags & FAULT_FLAG_TRIED) return; /* * This is needed only for protection faults but the arch code * is not yet telling us if this is a protection fault or not. * This still avoids useless tlb flushes for .text page faults * with threads. */ if (vmf->flags & FAULT_FLAG_WRITE) { if (ptlevel == PGTABLE_LEVEL_PTE) flush_tlb_fix_spurious_fault(vmf->vma, vmf->address, vmf->pte); else flush_tlb_fix_spurious_fault_pmd(vmf->vma, vmf->address, vmf->pmd); }}/* * These routines also need to handle stuff like marking pages dirty * and/or accessed for architectures that don't do it in hardware (most * RISC architectures). The early dirtying is also good on the i386. * * There is also a hook called "update_mmu_cache()" that architectures * with external mmu caches can use to update those (ie the Sparc or * PowerPC hashed page tables that act as extended TLBs). * * On entry, we hold either the VMA lock or the mmap_lock * (see FAULT_FLAG_VMA_LOCK). * * The mmap_lock or VMA lock may have been released depending on flags * and our return value. * See filemap_fault() and __folio_lock_or_retry(). */static vm_fault_t handle_pte_fault(struct vm_fault *vmf){ pte_t entry;
if (unlikely(pmd_none(*vmf->pmd))) { /* * Leave __pte_alloc() until later: because vm_ops->fault may * want to allocate huge page, and if we expose page table * for an instant, it will be difficult to retract from * concurrent faults and from rmap lookups. */ vmf->pte = NULL; vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID; } else { pmd_t dummy_pmdval;
/* * A regular pmd is established and it can't morph into a huge * pmd by anon khugepaged, since that takes mmap_lock in write * mode; but shmem or file collapse to THP could still morph * it into a huge pmd: just retry later if so. * * Use the maywrite version to indicate that vmf->pte may be * modified, but since we will use pte_same() to detect the * change of the !pte_none() entry, there is no need to recheck * the pmdval. Here we choose to pass a dummy variable instead * of NULL, which helps new user think about why this place is * special. */ vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd, vmf->address, &dummy_pmdval, &vmf->ptl); if (unlikely(!vmf->pte)) return 0; vmf->orig_pte = ptep_get_lockless(vmf->pte); vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
if (pte_none(vmf->orig_pte)) { pte_unmap(vmf->pte); vmf->pte = NULL; } }
if (!vmf->pte) return do_pte_missing(vmf);
if (!pte_present(vmf->orig_pte)) return do_swap_page(vmf);
if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) { /* * RWP-protected PTEs are protnone plus the uffd bit. On a * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is * NUMA hinting and must still fall through to do_numa_page(). */ if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) return do_uffd_rwp(vmf); return do_numa_page(vmf); }
spin_lock(vmf->ptl); entry = vmf->orig_pte; if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) { update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); goto unlock; } if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) { if (!pte_write(entry)) return do_wp_page(vmf); else if (likely(vmf->flags & FAULT_FLAG_WRITE)) entry = pte_mkdirty(entry); } entry = pte_mkyoung(entry); if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry, vmf->flags & FAULT_FLAG_WRITE)) update_mmu_cache_range(vmf, vmf->vma, vmf->address, vmf->pte, 1); else fix_spurious_fault(vmf, PGTABLE_LEVEL_PTE);unlock: pte_unmap_unlock(vmf->pte, vmf->ptl); return 0;}
/* * On entry, we hold either the VMA lock or the mmap_lock * (see FAULT_FLAG_VMA_LOCK). If VM_FAULT_RETRY is set in * the result, the lock is not held on exit. See filemap_fault() * and __folio_lock_or_retry(). */static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags){ struct vm_fault vmf = { .vma = vma, .address = address & PAGE_MASK, .real_address = address, .flags = flags, .pgoff = linear_page_index(vma, address), .gfp_mask = __get_fault_gfp_mask(vma), }; struct mm_struct *mm = vma->vm_mm; vm_flags_t vm_flags = vma->vm_flags; pgd_t *pgd; p4d_t *p4d; vm_fault_t ret;
pgd = pgd_offset(mm, address); p4d = p4d_alloc(mm, pgd, address); if (!p4d) return VM_FAULT_OOM;
vmf.pud = pud_alloc(mm, p4d, address); if (!vmf.pud) return VM_FAULT_OOM;retry_pud: if (pud_none(*vmf.pud) && thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PUD_ORDER)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; } else { pud_t orig_pud = *vmf.pud;
barrier(); if (pud_trans_huge(orig_pud)) {
/* * TODO once we support anonymous PUDs: NUMA case and * FAULT_FLAG_UNSHARE handling. */ if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) { ret = wp_huge_pud(&vmf, orig_pud); if (!(ret & VM_FAULT_FALLBACK)) return ret; } else { huge_pud_set_accessed(&vmf, orig_pud); return 0; } } }
vmf.pmd = pmd_alloc(mm, vmf.pud, address); if (!vmf.pmd) return VM_FAULT_OOM;
/* Huge pud page fault raced with pmd_alloc? */ if (pud_trans_unstable(vmf.pud)) goto retry_pud;
if (pmd_none(*vmf.pmd) && thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PMD_ORDER)) { ret = create_huge_pmd(&vmf); if (ret & VM_FAULT_FALLBACK) goto fallback; else return ret; }
vmf.orig_pmd = pmdp_get_lockless(vmf.pmd); if (pmd_none(vmf.orig_pmd)) goto fallback;
if (unlikely(!pmd_present(vmf.orig_pmd))) { if (pmd_is_device_private_entry(vmf.orig_pmd)) return do_huge_pmd_device_private(&vmf);
if (pmd_is_migration_entry(vmf.orig_pmd)) pmd_migration_entry_wait(mm, vmf.pmd); return 0; } if (pmd_trans_huge(vmf.orig_pmd)) { if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) { if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd)) return do_huge_pmd_uffd_rwp(&vmf); return do_huge_pmd_numa_page(&vmf); }
if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && !pmd_write(vmf.orig_pmd)) { ret = wp_huge_pmd(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; } else { vmf.ptl = pmd_lock(mm, vmf.pmd); if (!huge_pmd_set_accessed(&vmf)) fix_spurious_fault(&vmf, PGTABLE_LEVEL_PMD); spin_unlock(vmf.ptl); return 0; } }
fallback: return handle_pte_fault(&vmf);}
/** * mm_account_fault - Do page fault accounting * @mm: mm from which memcg should be extracted. It can be NULL. * @regs: the pt_regs struct pointer. When set to NULL, will skip accounting * of perf event counters, but we'll still do the per-task accounting to * the task who triggered this page fault. * @address: the faulted address. * @flags: the fault flags. * @ret: the fault retcode. * * This will take care of most of the page fault accounting. Meanwhile, it * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter * updates. However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should * still be in per-arch page fault handlers at the entry of page fault. */static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs, unsigned long address, unsigned int flags, vm_fault_t ret){ bool major;
/* Incomplete faults will be accounted upon completion. */ if (ret & VM_FAULT_RETRY) return;
/* * To preserve the behavior of older kernels, PGFAULT counters record * both successful and failed faults, as opposed to perf counters, * which ignore failed cases. */ count_vm_event(PGFAULT); count_memcg_event_mm(mm, PGFAULT);
/* * Do not account for unsuccessful faults (e.g. when the address wasn't * valid). That includes arch_vma_access_permitted() failing before * reaching here. So this is not a "this many hardware page faults" * counter. We should use the hw profiling for that. */ if (ret & VM_FAULT_ERROR) return;
/* * We define the fault as a major fault when the final successful fault * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't * handle it immediately previously). */ major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
if (major) current->maj_flt++; else current->min_flt++;
/* * If the fault is done for GUP, regs will be NULL. We only do the * accounting for the per thread fault counters who triggered the * fault, and we skip the perf event updates. */ if (!regs) return;
if (major) perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); else perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);}
#ifdef CONFIG_LRU_GENstatic void lru_gen_enter_fault(struct vm_area_struct *vma){ /* the LRU algorithm only applies to accesses with recency */ current->in_lru_fault = vma_has_recency(vma);}
static void lru_gen_exit_fault(void){ current->in_lru_fault = false;}#elsestatic void lru_gen_enter_fault(struct vm_area_struct *vma){}
static void lru_gen_exit_fault(void){}#endif /* CONFIG_LRU_GEN */
static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma, unsigned int *flags){ if (unlikely(*flags & FAULT_FLAG_UNSHARE)) { if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE)) return VM_FAULT_SIGSEGV; /* * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's * just treat it like an ordinary read-fault otherwise. */ if (!vma_is_cow_mapping(vma)) *flags &= ~FAULT_FLAG_UNSHARE; } else if (*flags & FAULT_FLAG_WRITE) { /* Write faults on read-only mappings are impossible ... */ if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE))) return VM_FAULT_SIGSEGV; /* ... and FOLL_FORCE only applies to COW mappings. */ if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) && !vma_is_cow_mapping(vma))) return VM_FAULT_SIGSEGV; } /* * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of * the assumption that lock is dropped on VM_FAULT_RETRY. */ if (WARN_ON_ONCE((*flags & (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) == (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT))) return VM_FAULT_SIGSEGV;
return 0;}
/* * By the time we get here, we already hold either the VMA lock or the * mmap_lock (see FAULT_FLAG_VMA_LOCK). * * The lock may have been released depending on flags and our * return value. See filemap_fault() and __folio_lock_or_retry(). */vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags, struct pt_regs *regs){ /* If the fault handler drops the mmap_lock, vma may be freed */ struct mm_struct *mm = vma->vm_mm; vm_fault_t ret; bool is_droppable;
__set_current_state(TASK_RUNNING);
ret = sanitize_fault_flags(vma, &flags); if (ret) goto out;
if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, flags & FAULT_FLAG_INSTRUCTION, flags & FAULT_FLAG_REMOTE)) { ret = VM_FAULT_SIGSEGV; goto out; }
is_droppable = !!(vma->vm_flags & VM_DROPPABLE);
/* * Enable the memcg OOM handling for faults triggered in user * space. Kernel faults are handled more gracefully. */ if (flags & FAULT_FLAG_USER) mem_cgroup_enter_user_fault();
lru_gen_enter_fault(vma);
if (unlikely(vma_is_hugetlb(vma))) ret = hugetlb_fault(vma->vm_mm, vma, address, flags); else ret = __handle_mm_fault(vma, address, flags);
/* * Warning: It is no longer safe to dereference vma-> after this point, * because mmap_lock might have been dropped by __handle_mm_fault(), so * vma might be destroyed from underneath us. */
lru_gen_exit_fault();
/* If the mapping is droppable, then errors due to OOM aren't fatal. */ if (is_droppable) ret &= ~VM_FAULT_OOM;
if (flags & FAULT_FLAG_USER) { mem_cgroup_exit_user_fault(); /* * The task may have entered a memcg OOM situation but * if the allocation error was handled gracefully (no * VM_FAULT_OOM), there is no need to kill anything. * Just clean up the OOM state peacefully. */ if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM)) mem_cgroup_oom_synchronize(false); }out: mm_account_fault(mm, regs, address, flags, ret);
return ret;}EXPORT_SYMBOL_GPL(handle_mm_fault);
#ifndef __PAGETABLE_P4D_FOLDED/* * Allocate p4d page table. * We've already handled the fast-path in-line. */int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address){ p4d_t *new = p4d_alloc_one(mm, address); if (!new) return -ENOMEM;
spin_lock(&mm->page_table_lock); if (pgd_present(*pgd)) { /* Another has populated it */ p4d_free(mm, new); } else { smp_wmb(); /* See comment in pmd_install() */ pgd_populate(mm, pgd, new); } spin_unlock(&mm->page_table_lock); return 0;}#endif /* __PAGETABLE_P4D_FOLDED */
#ifndef __PAGETABLE_PUD_FOLDED/* * Allocate page upper directory. * We've already handled the fast-path in-line. */int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address){ pud_t *new = pud_alloc_one(mm, address); if (!new) return -ENOMEM;
spin_lock(&mm->page_table_lock); if (!p4d_present(*p4d)) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); } else /* Another has populated it */ pud_free(mm, new); spin_unlock(&mm->page_table_lock); return 0;}#endif /* __PAGETABLE_PUD_FOLDED */
#ifndef __PAGETABLE_PMD_FOLDED/* * Allocate page middle directory. * We've already handled the fast-path in-line. */int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address){ spinlock_t *ptl; pmd_t *new = pmd_alloc_one(mm, address); if (!new) return -ENOMEM;
ptl = pud_lock(mm, pud); if (!pud_present(*pud)) { mm_inc_nr_pmds(mm); smp_wmb(); /* See comment in pmd_install() */ pud_populate(mm, pud, new); } else { /* Another has populated it */ pmd_free(mm, new); } spin_unlock(ptl); return 0;}#endif /* __PAGETABLE_PMD_FOLDED */
static inline void pfnmap_args_setup(struct follow_pfnmap_args *args, spinlock_t *lock, pte_t *ptep, pgprot_t pgprot, unsigned long pfn_base, unsigned long addr_mask, bool writable, bool special){ args->lock = lock; args->ptep = ptep; args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); args->addr_mask = addr_mask; args->pgprot = pgprot; args->writable = writable; args->special = special;}
static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma){#ifdef CONFIG_LOCKDEP struct file *file = vma->vm_file; struct address_space *mapping = file ? file->f_mapping : NULL;
if (mapping) lockdep_assert(lockdep_is_held(&mapping->i_mmap_rwsem) || lockdep_is_held(&vma->vm_mm->mmap_lock)); else lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));#endif}
/** * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address * @args: Pointer to struct @follow_pfnmap_args * * The caller needs to setup args->vma and args->address to point to the * virtual address as the target of such lookup. On a successful return, * the results will be put into other output fields. * * After the caller finished using the fields, the caller must invoke * another follow_pfnmap_end() to proper releases the locks and resources * of such look up request. * * During the start() and end() calls, the results in @args will be valid * as proper locks will be held. After the end() is called, all the fields * in @follow_pfnmap_args will be invalid to be further accessed. Further * use of such information after end() may require proper synchronizations * by the caller with page table updates, otherwise it can create a * security bug. * * If the PTE maps a refcounted page, callers are responsible to protect * against invalidation with MMU notifiers; otherwise access to the PFN at * a later point in time can trigger use-after-free. * * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore * should be taken for read, and the mmap semaphore cannot be released * before the end() is invoked. * * This function must not be used to modify PTE content. * * Return: zero on success, negative otherwise. */int follow_pfnmap_start(struct follow_pfnmap_args *args){ struct vm_area_struct *vma = args->vma; unsigned long address = args->address; struct mm_struct *mm = vma->vm_mm; spinlock_t *lock; pgd_t *pgdp; p4d_t *p4dp, p4d; pud_t *pudp, pud; pmd_t *pmdp, pmd; pte_t *ptep, pte;
pfnmap_lockdep_assert(vma);
if (unlikely(address < vma->vm_start || address >= vma->vm_end)) goto out;
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out;retry: pgdp = pgd_offset(mm, address); if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) goto out;
p4dp = p4d_offset(pgdp, address); p4d = p4dp_get(p4dp); if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) goto out;
pudp = pud_offset(p4dp, address); pud = pudp_get(pudp); if (!pud_present(pud)) goto out; if (pud_leaf(pud)) { lock = pud_lock(mm, pudp); pud = pudp_get(pudp);
if (unlikely(!pud_present(pud))) { spin_unlock(lock); goto out; } else if (unlikely(!pud_leaf(pud))) { spin_unlock(lock); goto retry; } pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), pud_pfn(pud), PUD_MASK, pud_write(pud), pud_special(pud)); return 0; }
pmdp = pmd_offset(pudp, address); pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) goto out; if (pmd_leaf(pmd)) { lock = pmd_lock(mm, pmdp); pmd = pmdp_get(pmdp);
if (unlikely(!pmd_present(pmd))) { spin_unlock(lock); goto out; } else if (unlikely(!pmd_leaf(pmd))) { spin_unlock(lock); goto retry; } pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), pmd_special(pmd)); return 0; }
ptep = pte_offset_map_lock(mm, pmdp, address, &lock); if (!ptep) goto out; pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), pte_pfn(pte), PAGE_MASK, pte_write(pte), pte_special(pte)); return 0;unlock: pte_unmap_unlock(ptep, lock);out: return -EINVAL;}EXPORT_SYMBOL_GPL(follow_pfnmap_start);
/** * follow_pfnmap_end(): End a follow_pfnmap_start() process * @args: Pointer to struct @follow_pfnmap_args * * Must be used in pair of follow_pfnmap_start(). See the start() function * above for more information. */void follow_pfnmap_end(struct follow_pfnmap_args *args){ if (args->lock) spin_unlock(args->lock); if (args->ptep) pte_unmap(args->ptep);}EXPORT_SYMBOL_GPL(follow_pfnmap_end);
#ifdef CONFIG_HAVE_IOREMAP_PROT/** * generic_access_phys - generic implementation for iomem mmap access * @vma: the vma to access * @addr: userspace address, not relative offset within @vma * @buf: buffer to read/write * @len: length of transfer * @write: set to FOLL_WRITE when writing, otherwise reading * * This is a generic implementation for &vm_operations_struct.access for an * iomem mapping. This callback is used by access_process_vm() when the @vma is * not page based. */int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write){ resource_size_t phys_addr; pgprot_t prot = __pgprot(0); void __iomem *maddr; struct page *page; int offset = offset_in_page(addr); int ret = -EINVAL; bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr };
/* * Limit access to one page at a time, as that's what follow_pfnmap_start() * guarantees; expect the caller to retry to read larger ranges. */ len = min_t(int, len, PAGE_SIZE - offset);
retry: if (follow_pfnmap_start(&args)) return -EINVAL;
/* * Reject anything that looks like ordinary RAM, including anonymous * folios in CoW mappings or allocated kernel memory. Note that we * cannot distinguish memblock allocations from non-RAM reliably, so * we cannot easily reject them here. */ page = pfn_to_online_page(args.pfn); if (page && !PageReserved(page)) { follow_pfnmap_end(&args); return -EINVAL; }
prot = args.pgprot; phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT; writable = args.writable; follow_pfnmap_end(&args);
if ((write & FOLL_WRITE) && !writable) return -EINVAL;
maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); if (!maddr) return -ENOMEM;
if (follow_pfnmap_start(&args)) goto out_unmap;
if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); goto retry; }
if (write) memcpy_toio(maddr + offset, buf, len); else memcpy_fromio(buf, maddr + offset, len); ret = len; follow_pfnmap_end(&args);out_unmap: iounmap(maddr);
return ret;}EXPORT_SYMBOL_GPL(generic_access_phys);#endif
/* * Access another process' address space as given in mm. */static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags){ void *old_buf = buf; int write = gup_flags & FOLL_WRITE;
if (mmap_read_lock_killable(mm)) return 0;
/* Untag the address before looking up the VMA */ addr = untagged_addr_remote(mm, addr);
/* Avoid triggering the temporary warning in __get_user_pages */ if (!vma_lookup(mm, addr) && !expand_stack(mm, addr)) return 0;
/* ignore errors, just check how much was successfully transferred */ while (len) { int bytes, offset; void *maddr; struct folio *folio; struct vm_area_struct *vma = NULL; struct page *page = get_user_page_vma_remote(mm, addr, gup_flags, &vma);
if (IS_ERR(page)) { /* We might need to expand the stack to access it */ vma = vma_lookup(mm, addr); if (!vma) { vma = expand_stack(mm, addr);
/* mmap_lock was dropped on failure */ if (!vma) return buf - old_buf;
/* Try again if stack expansion worked */ continue; }
/* * Check if this is a VM_IO | VM_PFNMAP VMA, which * we can access using slightly different code. */ bytes = 0;#ifdef CONFIG_HAVE_IOREMAP_PROT if (vma->vm_ops && vma->vm_ops->access) bytes = vma->vm_ops->access(vma, addr, buf, len, write);#endif if (bytes <= 0) break; } else { folio = page_folio(page); bytes = len; offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset;
maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); if (write) { copy_to_user_page(vma, page, addr, maddr + offset, buf, bytes); folio_mark_dirty_lock(folio); } else { copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } folio_release_kmap(folio, maddr); } len -= bytes; buf += bytes; addr += bytes; } mmap_read_unlock(mm);
return buf - old_buf;}
/** * access_remote_vm - access another process' address space * @mm: the mm_struct of the target address space * @addr: start address to access * @buf: source or destination buffer * @len: number of bytes to transfer * @gup_flags: flags modifying lookup behaviour * * The caller must hold a reference on @mm. * * Return: number of bytes copied from source to destination. */int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags){ return __access_remote_vm(mm, addr, buf, len, gup_flags);}
/* * Access another process' address space. * Source/target buffer must be kernel space, * Do not walk the page table directly, use get_user_pages */int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags){ struct mm_struct *mm; int ret;
mm = get_task_mm(tsk); if (!mm) return 0;
ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
mmput(mm);
return ret;}EXPORT_SYMBOL_GPL(access_process_vm);
/* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags){ void *old_buf = buf; int err = 0;
*(char *)buf = '\0';
if (mmap_read_lock_killable(mm)) return -EFAULT;
addr = untagged_addr_remote(mm, addr);
/* Avoid triggering the temporary warning in __get_user_pages */ if (!vma_lookup(mm, addr)) { err = -EFAULT; goto out; }
while (len) { int bytes, offset, retval; void *maddr; struct folio *folio; struct page *page; struct vm_area_struct *vma = NULL;
page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { /* * Treat as a total failure for now until we decide how * to handle the CONFIG_HAVE_IOREMAP_PROT case and * stack expansion. */ *(char *)buf = '\0'; err = -EFAULT; goto out; }
folio = page_folio(page); bytes = len; offset = addr & (PAGE_SIZE - 1); if (bytes > PAGE_SIZE - offset) bytes = PAGE_SIZE - offset;
maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); retval = strscpy(buf, maddr + offset, bytes); if (retval >= 0) { /* Found the end of the string */ buf += retval; folio_release_kmap(folio, maddr); break; }
buf += bytes - 1; /* * Because strscpy always NUL terminates we need to * copy the last byte in the page if we are going to * load more pages */ if (bytes != len) { addr += bytes - 1; copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1); buf += 1; addr += 1; } len -= bytes;
folio_release_kmap(folio, maddr); }
out: mmap_read_unlock(mm); if (err) return err; return buf - old_buf;}
/* * Print the name of a VMA. */void print_vma_addr(char *prefix, unsigned long ip){ struct mm_struct *mm = current->mm; struct vm_area_struct *vma;
/* * we might be running from an atomic context so we cannot sleep */ if (!mmap_read_trylock(mm)) return;
vma = vma_lookup(mm, ip); if (vma && vma->vm_file) { struct file *f = vma->vm_file; ip -= vma->vm_start; ip += vma_start_pgoff(vma) << PAGE_SHIFT; printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip, vma->vm_start, vma->vm_end - vma->vm_start); } mmap_read_unlock(mm);}
#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)void __might_fault(const char *file, int line){ if (pagefault_disabled()) return; __might_sleep(file, line); if (current->mm) might_lock_read(¤t->mm->mmap_lock);}EXPORT_SYMBOL(__might_fault);#endif
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)/* * Process all subpages of the specified huge page with the specified * operation. The target subpage will be processed last to keep its * cache lines hot. */static inline int process_huge_page( unsigned long addr_hint, unsigned int nr_pages, int (*process_subpage)(unsigned long addr, int idx, void *arg), void *arg){ int i, n, base, l, ret; unsigned long addr = addr_hint & ~(((unsigned long)nr_pages << PAGE_SHIFT) - 1);
/* Process target subpage last to keep its cache lines hot */ might_sleep(); n = (addr_hint - addr) / PAGE_SIZE; if (2 * n <= nr_pages) { /* If target subpage in first half of huge page */ base = 0; l = n; /* Process subpages at the end of huge page */ for (i = nr_pages - 1; i >= 2 * n; i--) { cond_resched(); ret = process_subpage(addr + i * PAGE_SIZE, i, arg); if (ret) return ret; } } else { /* If target subpage in second half of huge page */ base = nr_pages - 2 * (nr_pages - n); l = nr_pages - n; /* Process subpages at the begin of huge page */ for (i = 0; i < base; i++) { cond_resched(); ret = process_subpage(addr + i * PAGE_SIZE, i, arg); if (ret) return ret; } } /* * Process remaining subpages in left-right-left-right pattern * towards the target subpage */ for (i = 0; i < l; i++) { int left_idx = base + i; int right_idx = base + 2 * l - 1 - i;
cond_resched(); ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg); if (ret) return ret; cond_resched(); ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg); if (ret) return ret; } return 0;}
static void clear_contig_highpages(struct page *page, unsigned long addr, unsigned int nr_pages){ unsigned int i, count; /* * When clearing we want to operate on the largest extent possible to * allow for architecture specific extent based optimizations. * * However, since clear_user_highpages() (and primitives clear_user_pages(), * clear_pages()), do not call cond_resched(), limit the unit size when * running under non-preemptible scheduling models. */ const unsigned int unit = preempt_model_preemptible() ? nr_pages : PROCESS_PAGES_NON_PREEMPT_BATCH;
might_sleep();
for (i = 0; i < nr_pages; i += count) { cond_resched();
count = min(unit, nr_pages - i); clear_user_highpages(page + i, addr + i * PAGE_SIZE, count); }}
/* * When zeroing a folio, we want to differentiate between pages in the * vicinity of the faulting address where we have spatial and temporal * locality, and those far away where we don't. * * Use a radius of 2 for determining the local neighbourhood. */#define FOLIO_ZERO_LOCALITY_RADIUS 2
/** * folio_zero_user - Zero a folio which will be mapped to userspace. * @folio: The folio to zero. * @addr_hint: The address accessed by the user or the base address. */void folio_zero_user(struct folio *folio, unsigned long addr_hint){ const unsigned long base_addr = ALIGN_DOWN(addr_hint, folio_size(folio)); const long fault_idx = (addr_hint - base_addr) / PAGE_SIZE; const struct range pg = DEFINE_RANGE(0, folio_nr_pages(folio) - 1); const long radius = FOLIO_ZERO_LOCALITY_RADIUS; struct range r[3]; int i;
/* * Faulting page and its immediate neighbourhood. Will be cleared at the * end to keep its cachelines hot. */ r[2] = DEFINE_RANGE(fault_idx - radius < (long)pg.start ? pg.start : fault_idx - radius, fault_idx + radius > (long)pg.end ? pg.end : fault_idx + radius);
/* Region to the left of the fault */ r[1] = DEFINE_RANGE(pg.start, r[2].start - 1);
/* Region to the right of the fault: always valid for the common fault_idx=0 case. */ r[0] = DEFINE_RANGE(r[2].end + 1, pg.end);
for (i = 0; i < ARRAY_SIZE(r); i++) { const unsigned long addr = base_addr + r[i].start * PAGE_SIZE; const long nr_pages = (long)range_len(&r[i]); struct page *page = folio_page(folio, r[i].start);
if (nr_pages > 0) clear_contig_highpages(page, addr, nr_pages); }}
static int copy_user_gigantic_page(struct folio *dst, struct folio *src, unsigned long addr_hint, struct vm_area_struct *vma, unsigned int nr_pages){ unsigned long addr = ALIGN_DOWN(addr_hint, folio_size(dst)); struct page *dst_page; struct page *src_page; int i;
for (i = 0; i < nr_pages; i++) { dst_page = folio_page(dst, i); src_page = folio_page(src, i);
cond_resched(); if (copy_mc_user_highpage(dst_page, src_page, addr + i*PAGE_SIZE, vma)) return -EHWPOISON; } return 0;}
struct copy_subpage_arg { struct folio *dst; struct folio *src; struct vm_area_struct *vma;};
static int copy_subpage(unsigned long addr, int idx, void *arg){ struct copy_subpage_arg *copy_arg = arg; struct page *dst = folio_page(copy_arg->dst, idx); struct page *src = folio_page(copy_arg->src, idx);
if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma)) return -EHWPOISON; return 0;}
int copy_user_large_folio(struct folio *dst, struct folio *src, unsigned long addr_hint, struct vm_area_struct *vma){ unsigned int nr_pages = folio_nr_pages(dst); struct copy_subpage_arg arg = { .dst = dst, .src = src, .vma = vma, };
if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages);
return process_huge_page(addr_hint, nr_pages, copy_subpage, &arg);}
long copy_folio_from_user(struct folio *dst_folio, const void __user *usr_src, bool allow_pagefault){ void *kaddr; unsigned long i, rc = 0; unsigned int nr_pages = folio_nr_pages(dst_folio); unsigned long ret_val = nr_pages * PAGE_SIZE; struct page *subpage;
for (i = 0; i < nr_pages; i++) { subpage = folio_page(dst_folio, i); kaddr = kmap_local_page(subpage); if (!allow_pagefault) pagefault_disable(); rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE); if (!allow_pagefault) pagefault_enable(); kunmap_local(kaddr);
ret_val -= (PAGE_SIZE - rc); if (rc) break;
flush_dcache_page(subpage);
cond_resched(); } return ret_val;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
#if defined(CONFIG_SPLIT_PTE_PTLOCKS) && ALLOC_SPLIT_PTLOCKS
static struct kmem_cache *page_ptl_cachep;
void __init ptlock_cache_init(void){ page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0, SLAB_PANIC, NULL);}
bool ptlock_alloc(struct ptdesc *ptdesc){ spinlock_t *ptl;
ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL); if (!ptl) return false; ptdesc->ptl = ptl; return true;}
void ptlock_free(struct ptdesc *ptdesc){ if (ptdesc->ptl) kmem_cache_free(page_ptl_cachep, ptdesc->ptl);}#endif
void vma_pgtable_walk_begin(struct vm_area_struct *vma){ if (vma_is_hugetlb(vma)) hugetlb_vma_lock_read(vma);}
void vma_pgtable_walk_end(struct vm_area_struct *vma){ if (vma_is_hugetlb(vma)) hugetlb_vma_unlock_read(vma);}