Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_MM_H#define _LINUX_MM_H
#include <linux/args.h>#include <linux/errno.h>#include <linux/mmdebug.h>#include <linux/gfp.h>#include <linux/pgalloc_tag.h>#include <linux/bug.h>#include <linux/list.h>#include <linux/mmzone.h>#include <linux/rbtree.h>#include <linux/atomic.h>#include <linux/debug_locks.h>#include <linux/compiler.h>#include <linux/mm_types.h>#include <linux/mmap_lock.h>#include <linux/range.h>#include <linux/pfn.h>#include <linux/percpu-refcount.h>#include <linux/bit_spinlock.h>#include <linux/shrinker.h>#include <linux/resource.h>#include <linux/page_ext.h>#include <linux/err.h>#include <linux/page-flags.h>#include <linux/page_ref.h>#include <linux/overflow.h>#include <linux/sched.h>#include <linux/pgtable.h>#include <linux/kasan.h>#include <linux/memremap.h>#include <linux/slab.h>#include <linux/cacheinfo.h>#include <linux/rcuwait.h>#include <linux/bitmap.h>#include <linux/bitops.h>#include <linux/iommu-debug-pagealloc.h>
struct mempolicy;struct anon_vma;struct anon_vma_chain;struct user_struct;struct pt_regs;struct folio_batch;
void arch_mm_preinit(void);void mm_core_init_early(void);void mm_core_init(void);void init_mm_internals(void);
extern atomic_long_t _totalram_pages;static inline unsigned long totalram_pages(void){ return (unsigned long)atomic_long_read(&_totalram_pages);}
static inline void totalram_pages_inc(void){ atomic_long_inc(&_totalram_pages);}
static inline void totalram_pages_dec(void){ atomic_long_dec(&_totalram_pages);}
static inline void totalram_pages_add(long count){ atomic_long_add(count, &_totalram_pages);}
extern void * high_memory;
/* * Convert between pages and MB * 20 is the shift for 1MB (2^20 = 1MB) * PAGE_SHIFT is the shift for page size (e.g., 12 for 4KB pages) * So (20 - PAGE_SHIFT) converts between pages and MB */#define PAGES_TO_MB(pages) ((pages) >> (20 - PAGE_SHIFT))#define MB_TO_PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
#ifdef CONFIG_SYSCTLextern int sysctl_legacy_va_layout;#else#define sysctl_legacy_va_layout 0#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITSextern const int mmap_rnd_bits_min;extern int mmap_rnd_bits_max __ro_after_init;extern int mmap_rnd_bits __read_mostly;#endif#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITSextern const int mmap_rnd_compat_bits_min;extern const int mmap_rnd_compat_bits_max;extern int mmap_rnd_compat_bits __read_mostly;#endif
#ifndef DIRECT_MAP_PHYSMEM_END# ifdef MAX_PHYSMEM_BITS# define DIRECT_MAP_PHYSMEM_END ((1ULL << MAX_PHYSMEM_BITS) - 1)# else# define DIRECT_MAP_PHYSMEM_END (((phys_addr_t)-1)&~(1ULL<<63))# endif#endif
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
#include <asm/page.h>#include <asm/processor.h>
#ifndef __pa_symbol#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))#endif
#ifndef page_to_virt#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))#endif
#ifndef lm_alias#define lm_alias(x) __va(__pa_symbol(x))#endif
/* * To prevent common memory management code establishing * a zero page mapping on a read fault. * This macro should be defined within <asm/pgtable.h>. * s390 does this to prevent multiplexing of hardware bits * related to the physical page in case of virtualization. */#ifndef mm_forbids_zeropage#define mm_forbids_zeropage(X) (0)#endif
/* * On some architectures it is expensive to call memset() for small sizes. * If an architecture decides to implement their own version of * mm_zero_struct_page they should wrap the defines below in a #ifndef and * define their own version of this macro in <asm/pgtable.h> */#if BITS_PER_LONG == 64/* This function must be updated when the size of struct page grows above 96 * or reduces below 56. The idea that compiler optimizes out switch() * statement, and only leaves move/store instructions. Also the compiler can * combine write statements if they are both assignments and can be reordered, * this can result in several of the writes here being dropped. */#define mm_zero_struct_page(pp) __mm_zero_struct_page(pp)static inline void __mm_zero_struct_page(struct page *page){ unsigned long *_pp = (void *)page;
/* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ BUILD_BUG_ON(sizeof(struct page) & 7); BUILD_BUG_ON(sizeof(struct page) < 56); BUILD_BUG_ON(sizeof(struct page) > 96);
switch (sizeof(struct page)) { case 96: _pp[11] = 0; fallthrough; case 88: _pp[10] = 0; fallthrough; case 80: _pp[9] = 0; fallthrough; case 72: _pp[8] = 0; fallthrough; case 64: _pp[7] = 0; fallthrough; case 56: _pp[6] = 0; _pp[5] = 0; _pp[4] = 0; _pp[3] = 0; _pp[2] = 0; _pp[1] = 0; _pp[0] = 0; }}#else#define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))#endif
/* * Default maximum number of active map areas, this limits the number of vmas * per mm struct. Users can overwrite this number by sysctl but there is a * problem. * * When a program's coredump is generated as ELF format, a section is created * per a vma. In ELF, the number of sections is represented in unsigned short. * This means the number of sections should be smaller than 65535 at coredump. * Because the kernel adds some informative sections to a image of program at * generating coredump, we need some margin. The number of extra sections is * 1-3 now and depends on arch. We use "5" as safe margin, here. * * ELF extended numbering allows more than 65535 sections, so 16-bit bound is * not a hard limit any more. Although some userspace tools can be surprised by * that. */#define MAPCOUNT_ELF_CORE_MARGIN (5)#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
extern unsigned long sysctl_user_reserve_kbytes;extern unsigned long sysctl_admin_reserve_kbytes;
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)bool page_range_contiguous(const struct page *page, unsigned long nr_pages);#elsestatic inline bool page_range_contiguous(const struct page *page, unsigned long nr_pages){ return true;}#endif
/* to align the pointer to the (next) page boundary */#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
/* to align the pointer to the (prev) page boundary */#define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE)
/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
/** * folio_page_idx - Return the number of a page in a folio. * @folio: The folio. * @page: The folio page. * * This function expects that the page is actually part of the folio. * The returned number is relative to the start of the folio. */static inline unsigned long folio_page_idx(const struct folio *folio, const struct page *page){ return page - &folio->page;}
static inline struct folio *lru_to_folio(struct list_head *head){ return list_entry((head)->prev, struct folio, lru);}
void setup_initial_init_mm(void *start_code, void *end_code, void *end_data, void *brk);
/* * Linux kernel virtual memory manager primitives. * The idea being to have a "virtual" mm in the same way * we have a virtual fs - giving a cleaner interface to the * mm details, and allowing different kinds of memory mappings * (from shared memory to executable loading to arbitrary * mmap() functions). */
struct vm_area_struct *vm_area_alloc(struct mm_struct *);struct vm_area_struct *vm_area_dup(struct vm_area_struct *);void vm_area_free(struct vm_area_struct *);
#ifndef CONFIG_MMUextern struct rb_root nommu_region_tree;extern struct rw_semaphore nommu_region_sem;
extern unsigned int kobjsize(const void *objp);#endif
/* * vm_flags in vm_area_struct, see mm_types.h. * When changing, update also include/trace/events/mmflags.h */
#define VM_NONE 0x00000000
/** * typedef vma_flag_t - specifies an individual VMA flag by bit number. * * This value is made type safe by sparse to avoid passing invalid flag values * around. */typedef int __bitwise vma_flag_t;
#define DECLARE_VMA_BIT(name, bitnum) \ VMA_ ## name ## _BIT = ((__force vma_flag_t)bitnum)#define DECLARE_VMA_BIT_ALIAS(name, aliased) \ VMA_ ## name ## _BIT = (VMA_ ## aliased ## _BIT)enum { DECLARE_VMA_BIT(READ, 0), DECLARE_VMA_BIT(WRITE, 1), DECLARE_VMA_BIT(EXEC, 2), DECLARE_VMA_BIT(SHARED, 3), /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ DECLARE_VMA_BIT(MAYREAD, 4), /* limits for mprotect() etc. */ DECLARE_VMA_BIT(MAYWRITE, 5), DECLARE_VMA_BIT(MAYEXEC, 6), DECLARE_VMA_BIT(MAYSHARE, 7), DECLARE_VMA_BIT(GROWSDOWN, 8), /* general info on the segment */#ifdef CONFIG_MMU DECLARE_VMA_BIT(UFFD_MISSING, 9),/* missing pages tracking */#else /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ DECLARE_VMA_BIT(MAYOVERLAY, 9),#endif /* CONFIG_MMU */ /* Page-ranges managed without "struct page", just pure PFN */ DECLARE_VMA_BIT(PFNMAP, 10), DECLARE_VMA_BIT(MAYBE_GUARD, 11), DECLARE_VMA_BIT(UFFD_WP, 12), /* wrprotect pages tracking */ DECLARE_VMA_BIT(LOCKED, 13), DECLARE_VMA_BIT(IO, 14), /* Memory mapped I/O or similar */ DECLARE_VMA_BIT(SEQ_READ, 15), /* App will access data sequentially */ DECLARE_VMA_BIT(RAND_READ, 16), /* App will not benefit from clustered reads */ DECLARE_VMA_BIT(DONTCOPY, 17), /* Do not copy this vma on fork */ DECLARE_VMA_BIT(DONTEXPAND, 18),/* Cannot expand with mremap() */ DECLARE_VMA_BIT(LOCKONFAULT, 19),/* Lock pages covered when faulted in */ DECLARE_VMA_BIT(ACCOUNT, 20), /* Is a VM accounted object */ DECLARE_VMA_BIT(NORESERVE, 21), /* should the VM suppress accounting */ DECLARE_VMA_BIT(HUGETLB, 22), /* Huge TLB Page VM */ DECLARE_VMA_BIT(SYNC, 23), /* Synchronous page faults */ DECLARE_VMA_BIT(ARCH_1, 24), /* Architecture-specific flag */ DECLARE_VMA_BIT(WIPEONFORK, 25),/* Wipe VMA contents in child. */ DECLARE_VMA_BIT(DONTDUMP, 26), /* Do not include in the core dump */ DECLARE_VMA_BIT(SOFTDIRTY, 27), /* NOT soft dirty clean area */ DECLARE_VMA_BIT(MIXEDMAP, 28), /* Can contain struct page and pure PFN pages */ DECLARE_VMA_BIT(HUGEPAGE, 29), /* MADV_HUGEPAGE marked this vma */ DECLARE_VMA_BIT(NOHUGEPAGE, 30),/* MADV_NOHUGEPAGE marked this vma */ DECLARE_VMA_BIT(MERGEABLE, 31), /* KSM may merge identical pages */ /* These bits are reused, we define specific uses below. */ DECLARE_VMA_BIT(HIGH_ARCH_0, 32), DECLARE_VMA_BIT(HIGH_ARCH_1, 33), DECLARE_VMA_BIT(HIGH_ARCH_2, 34), DECLARE_VMA_BIT(HIGH_ARCH_3, 35), DECLARE_VMA_BIT(HIGH_ARCH_4, 36), DECLARE_VMA_BIT(HIGH_ARCH_5, 37), DECLARE_VMA_BIT(HIGH_ARCH_6, 38), /* * This flag is used to connect VFIO to arch specific KVM code. It * indicates that the memory under this VMA is safe for use with any * non-cachable memory type inside KVM. Some VFIO devices, on some * platforms, are thought to be unsafe and can cause machine crashes * if KVM does not lock down the memory type. */ DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39),#if defined(CONFIG_PPC32) DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1),#elif defined(CONFIG_64BIT) DECLARE_VMA_BIT(DROPPABLE, 40),#endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), DECLARE_VMA_BIT_ALIAS(PKEY_BIT2, HIGH_ARCH_2), DECLARE_VMA_BIT_ALIAS(PKEY_BIT3, HIGH_ARCH_3), DECLARE_VMA_BIT_ALIAS(PKEY_BIT4, HIGH_ARCH_4),#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) /* * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of * support core mm. * * These VMAs will get a single end guard page. This helps userspace * protect itself from attacks. A single page is enough for current * shadow stack archs (x86). See the comments near alloc_shstk() in * arch/x86/kernel/shstk.c for more details on the guard size. */ DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_5),#elif defined(CONFIG_ARM64_GCS) /* * arm64's Guarded Control Stack implements similar functionality and * has similar constraints to shadow stacks. */ DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_6),#endif DECLARE_VMA_BIT_ALIAS(SAO, ARCH_1), /* Strong Access Ordering (powerpc) */ DECLARE_VMA_BIT_ALIAS(GROWSUP, ARCH_1), /* parisc */ DECLARE_VMA_BIT_ALIAS(SPARC_ADI, ARCH_1), /* sparc64 */ DECLARE_VMA_BIT_ALIAS(ARM64_BTI, ARCH_1), /* arm64 */ DECLARE_VMA_BIT_ALIAS(ARCH_CLEAR, ARCH_1), /* sparc64, arm64 */ DECLARE_VMA_BIT_ALIAS(MAPPED_COPY, ARCH_1), /* !CONFIG_MMU */ DECLARE_VMA_BIT_ALIAS(MTE, HIGH_ARCH_4), /* arm64 */ DECLARE_VMA_BIT_ALIAS(MTE_ALLOWED, HIGH_ARCH_5),/* arm64 */#ifdef CONFIG_STACK_GROWSUP DECLARE_VMA_BIT_ALIAS(STACK, GROWSUP), DECLARE_VMA_BIT_ALIAS(STACK_EARLY, GROWSDOWN),#else DECLARE_VMA_BIT_ALIAS(STACK, GROWSDOWN),#endif};#undef DECLARE_VMA_BIT#undef DECLARE_VMA_BIT_ALIAS
#define INIT_VM_FLAG(name) BIT((__force int) VMA_ ## name ## _BIT)#define VM_READ INIT_VM_FLAG(READ)#define VM_WRITE INIT_VM_FLAG(WRITE)#define VM_EXEC INIT_VM_FLAG(EXEC)#define VM_SHARED INIT_VM_FLAG(SHARED)#define VM_MAYREAD INIT_VM_FLAG(MAYREAD)#define VM_MAYWRITE INIT_VM_FLAG(MAYWRITE)#define VM_MAYEXEC INIT_VM_FLAG(MAYEXEC)#define VM_MAYSHARE INIT_VM_FLAG(MAYSHARE)#define VM_GROWSDOWN INIT_VM_FLAG(GROWSDOWN)#ifdef CONFIG_MMU#define VM_UFFD_MISSING INIT_VM_FLAG(UFFD_MISSING)#else#define VM_UFFD_MISSING VM_NONE#define VM_MAYOVERLAY INIT_VM_FLAG(MAYOVERLAY)#endif#define VM_PFNMAP INIT_VM_FLAG(PFNMAP)#define VM_MAYBE_GUARD INIT_VM_FLAG(MAYBE_GUARD)#define VM_UFFD_WP INIT_VM_FLAG(UFFD_WP)#define VM_LOCKED INIT_VM_FLAG(LOCKED)#define VM_IO INIT_VM_FLAG(IO)#define VM_SEQ_READ INIT_VM_FLAG(SEQ_READ)#define VM_RAND_READ INIT_VM_FLAG(RAND_READ)#define VM_DONTCOPY INIT_VM_FLAG(DONTCOPY)#define VM_DONTEXPAND INIT_VM_FLAG(DONTEXPAND)#define VM_LOCKONFAULT INIT_VM_FLAG(LOCKONFAULT)#define VM_ACCOUNT INIT_VM_FLAG(ACCOUNT)#define VM_NORESERVE INIT_VM_FLAG(NORESERVE)#define VM_HUGETLB INIT_VM_FLAG(HUGETLB)#define VM_SYNC INIT_VM_FLAG(SYNC)#define VM_ARCH_1 INIT_VM_FLAG(ARCH_1)#define VM_WIPEONFORK INIT_VM_FLAG(WIPEONFORK)#define VM_DONTDUMP INIT_VM_FLAG(DONTDUMP)#ifdef CONFIG_MEM_SOFT_DIRTY#define VM_SOFTDIRTY INIT_VM_FLAG(SOFTDIRTY)#else#define VM_SOFTDIRTY VM_NONE#endif#define VM_MIXEDMAP INIT_VM_FLAG(MIXEDMAP)#define VM_HUGEPAGE INIT_VM_FLAG(HUGEPAGE)#define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE)#define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE)#define VM_STACK INIT_VM_FLAG(STACK)#ifdef CONFIG_STACK_GROWSUP#define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY)#else#define VM_STACK_EARLY VM_NONE#endif#ifdef CONFIG_ARCH_HAS_PKEYS#define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT)/* Despite the naming, these are FLAGS not bits. */#define VM_PKEY_BIT0 INIT_VM_FLAG(PKEY_BIT0)#define VM_PKEY_BIT1 INIT_VM_FLAG(PKEY_BIT1)#define VM_PKEY_BIT2 INIT_VM_FLAG(PKEY_BIT2)#if CONFIG_ARCH_PKEY_BITS > 3#define VM_PKEY_BIT3 INIT_VM_FLAG(PKEY_BIT3)#else#define VM_PKEY_BIT3 VM_NONE#endif /* CONFIG_ARCH_PKEY_BITS > 3 */#if CONFIG_ARCH_PKEY_BITS > 4#define VM_PKEY_BIT4 INIT_VM_FLAG(PKEY_BIT4)#else#define VM_PKEY_BIT4 VM_NONE#endif /* CONFIG_ARCH_PKEY_BITS > 4 */#endif /* CONFIG_ARCH_HAS_PKEYS */#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI)#define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK)#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT)#else#define VM_SHADOW_STACK VM_NONE#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT)#endif#if defined(CONFIG_PPC64)#define VM_SAO INIT_VM_FLAG(SAO)#elif defined(CONFIG_PARISC)#define VM_GROWSUP INIT_VM_FLAG(GROWSUP)#elif defined(CONFIG_SPARC64)#define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI)#define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR)#elif defined(CONFIG_ARM64)#define VM_ARM64_BTI INIT_VM_FLAG(ARM64_BTI)#define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR)#elif !defined(CONFIG_MMU)#define VM_MAPPED_COPY INIT_VM_FLAG(MAPPED_COPY)#endif#ifndef VM_GROWSUP#define VM_GROWSUP VM_NONE#endif#ifdef CONFIG_ARM64_MTE#define VM_MTE INIT_VM_FLAG(MTE)#define VM_MTE_ALLOWED INIT_VM_FLAG(MTE_ALLOWED)#else#define VM_MTE VM_NONE#define VM_MTE_ALLOWED VM_NONE#endif#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR#define VM_UFFD_MINOR INIT_VM_FLAG(UFFD_MINOR)#else#define VM_UFFD_MINOR VM_NONE#endif
/* * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose * bitmap cannot hold it. */#define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT)#define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT)#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR#define VMA_UFFD_MINOR mk_vma_flags(VMA_UFFD_MINOR_BIT)#else#define VMA_UFFD_MINOR EMPTY_VMA_FLAGS#endif
#ifdef CONFIG_64BIT#define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED)#define VM_SEALED INIT_VM_FLAG(SEALED)#else#define VM_ALLOW_ANY_UNCACHED VM_NONE#define VM_SEALED VM_NONE#endif#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)#define VM_DROPPABLE INIT_VM_FLAG(DROPPABLE)#define VMA_DROPPABLE mk_vma_flags(VMA_DROPPABLE_BIT)#else#define VM_DROPPABLE VM_NONE#define VMA_DROPPABLE EMPTY_VMA_FLAGS#endif
/* Bits set in the VMA until the stack is in its final location */#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
#define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VMA_EXEC_BIT : VMA_READ_BIT)
/* Common data flag combinations */#define VMA_DATA_FLAGS_TSK_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ TASK_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ VMA_MAYEXEC_BIT)#define VMA_DATA_FLAGS_NON_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT)#define VMA_DATA_FLAGS_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ VMA_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ VMA_MAYEXEC_BIT)
#ifndef VMA_DATA_DEFAULT_FLAGS /* arch can override this */#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_EXEC#endif
#ifndef VMA_STACK_DEFAULT_FLAGS /* arch can override this */#define VMA_STACK_DEFAULT_FLAGS VMA_DATA_DEFAULT_FLAGS#endif
#define VMA_STACK_FLAGS append_vma_flags(VMA_STACK_DEFAULT_FLAGS, \ VMA_STACK_BIT, VMA_ACCOUNT_BIT)
/* Temporary until VMA flags conversion complete. */#define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS)
#ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS#define VM_SEALED_SYSMAP VM_SEALED#else#define VM_SEALED_SYSMAP VM_NONE#endif
/* VMA basic access permission flags */#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)
/* * Special vmas that are non-mergable, non-mlock()able. */
#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \ VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)#define VM_SPECIAL vma_flags_to_legacy(VMA_SPECIAL_FLAGS)
/* * Physically remapped pages are special. Tell the * rest of the world about it: * IO tells people not to look at these pages * (accesses can have side effects). * PFNMAP tells the core MM that the base pages are just * raw PFN mappings, and do not have a "struct page" associated * with them. * DONTEXPAND * Disable vma merging and expanding with mremap(). * DONTDUMP * Omit vma from core dump, even when VM_IO turned off. */#define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, \ VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)
/* This mask prevents VMA from being scanned with khugepaged */#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
/* This mask defines which mm->def_flags a process can inherit its parent */#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
/* This mask represents all the VMA flag bits used by mlock */#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
#define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT)
/* These flags can be updated atomically via VMA/mmap read lock. */#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD
/* Arch-specific flags to clear when updating VM flags on protection change */#ifndef VM_ARCH_CLEAR#define VM_ARCH_CLEAR VM_NONE#endif#define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR)
/* * Flags which should be 'sticky' on merge - that is, flags which, when one VMA * possesses it but the other does not, the merged VMA should nonetheless have * applied to it: * * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its * references cleared via /proc/$pid/clear_refs, any * merged VMA should be considered soft-dirty also as it * operates at a VMA granularity. * * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies * that mapped page tables may contain metadata not * described by the VMA and thus any merged VMA may also * contain this metadata, and thus we must make this flag * sticky. */#ifdef CONFIG_MEM_SOFT_DIRTY#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)#else#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)#endif
/* * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one * of these flags and the other not does not preclude a merge. * * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they * are 'sticky'. If any sticky flags exist in either VMA, * we simply set all of them on the merged VMA. */#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS
/* * Flags which should result in page tables being copied on fork. These are * flags which indicate that the VMA maps page tables which cannot be * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the * source, as VM_UFFD_WP may not be propagated to destination, while all other * flags will be. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd * write protect handler, which cannot be * reconstructed on page fault. * * We always copy pgtables when dst_vma has uffd-wp * enabled even if it's file-backed * (e.g. shmem). Because when uffd-wp is enabled, * pgtable contains uffd-wp protection information, * that's something we can't retrieve from page cache, * and skip copying will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables * only and thus cannot be reconstructed on page * fault. */#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD)
/* * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask.. */
/* * The default fault flags that should be used by most of the * arch-specific page fault handlers. */#define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ FAULT_FLAG_KILLABLE | \ FAULT_FLAG_INTERRUPTIBLE)
/** * fault_flag_allow_retry_first - check ALLOW_RETRY the first time * @flags: Fault flags. * * This is mostly used for places where we want to try to avoid taking * the mmap_lock for too long a time when waiting for another condition * to change, in which case we can try to be polite to release the * mmap_lock in the first round to avoid potential starvation of other * processes that would also want the mmap_lock. * * Return: true if the page fault allows retry and this is the first * attempt of the fault handling; false otherwise. */static inline bool fault_flag_allow_retry_first(enum fault_flag flags){ return (flags & FAULT_FLAG_ALLOW_RETRY) && (!(flags & FAULT_FLAG_TRIED));}
#define FAULT_FLAG_TRACE \ { FAULT_FLAG_WRITE, "WRITE" }, \ { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ { FAULT_FLAG_TRIED, "TRIED" }, \ { FAULT_FLAG_USER, "USER" }, \ { FAULT_FLAG_REMOTE, "REMOTE" }, \ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" }
/* * vm_fault is filled by the pagefault handler and passed to the vma's * ->fault function. The vma's ->fault is responsible for returning a bitmask * of VM_FAULT_xxx flags that give details about how the fault was handled. * * MM layer fills up gfp_mask for page allocations but fault handler might * alter it if its implementation requires a different allocation context. * * pgoff should be used in favour of virtual_address, if possible. */struct vm_fault { const struct { struct vm_area_struct *vma; /* Target VMA */ gfp_t gfp_mask; /* gfp mask to be used for allocations */ pgoff_t pgoff; /* Logical page offset based on vma */ unsigned long address; /* Faulting virtual address - masked */ unsigned long real_address; /* Faulting virtual address - unmasked */ }; enum fault_flag flags; /* FAULT_FLAG_xxx flags * XXX: should really be 'const' */ pmd_t *pmd; /* Pointer to pmd entry matching * the 'address' */ pud_t *pud; /* Pointer to pud entry matching * the 'address' */ union { pte_t orig_pte; /* Value of PTE at the time of fault */ pmd_t orig_pmd; /* Value of PMD at the time of fault, * used by PMD fault only. */ };
struct page *cow_page; /* Page handler may use for COW fault */ struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by * VM_FAULT_ERROR). */ /* These three entries are valid only while holding ptl lock */ pte_t *pte; /* Pointer to pte entry matching * the 'address'. NULL if the page * table hasn't been allocated. */ spinlock_t *ptl; /* Page table lock. * Protects pte page table if 'pte' * is not NULL, otherwise pmd. */ pgtable_t prealloc_pte; /* Pre-allocated pte page table. * vm_ops->map_pages() sets up a page * table from atomic context. * do_fault_around() pre-allocates * page table to avoid allocation from * atomic context. */};
struct vm_uffd_ops;
/* * These are the virtual MM functions - opening of an area, closing and * unmapping it (needed to keep files on disk up-to-date etc), pointer * to the functions called when a no-page or a wp-page exception occurs. */struct vm_operations_struct { /** * @open: Called when a VMA is remapped, split or forked. Not called * upon first mapping a VMA. * Context: User context. May sleep. Caller holds mmap_lock. */ void (*open)(struct vm_area_struct *vma); /** * @close: Called when the VMA is being removed from the MM. * Context: User context. May sleep. Caller holds mmap_lock. */ void (*close)(struct vm_area_struct *vma); /** * @mapped: Called when the VMA is first mapped in the MM. Not called if * the new VMA is merged with an adjacent VMA. * * The @vm_private_data field is an output field allowing the user to * modify vma->vm_private_data as necessary. * * ONLY valid if set from f_op->mmap_prepare. Will result in an error if * set from f_op->mmap. * * Returns %0 on success, or an error otherwise. On error, the VMA will * be unmapped. * * Context: User context. May sleep. Caller holds mmap_lock. */ int (*mapped)(unsigned long start, unsigned long end, pgoff_t pgoff, const struct file *file, void **vm_private_data); /* Called any time before splitting to check if it's allowed */ int (*may_split)(struct vm_area_struct *vma, unsigned long addr); int (*mremap)(struct vm_area_struct *vma); /* * Called by mprotect() to make driver-specific permission * checks before mprotect() is finalised. The VMA must not * be modified. Returns 0 if mprotect() can proceed. */ int (*mprotect)(struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long newflags); vm_fault_t (*fault)(struct vm_fault *vmf); vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); vm_fault_t (*map_pages)(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); unsigned long (*pagesize)(struct vm_area_struct *vma);
/* notification that a previously read-only page is about to become * writable, if an error is returned it will cause a SIGBUS */ vm_fault_t (*page_mkwrite)(struct vm_fault *vmf);
/* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
/* called by access_process_vm when get_user_pages() fails, typically * for use by special VMAs. See also generic_access_phys() for a generic * implementation useful for any iomem mapping. */ int (*access)(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write);
/* Called by the /proc/PID/maps code to ask the vma whether it * has a special name. Returning non-NULL will also cause this * vma to be dumped unconditionally. */ const char *(*name)(struct vm_area_struct *vma);
#ifdef CONFIG_NUMA /* * set_policy() op must add a reference to any non-NULL @new mempolicy * to hold the policy upon return. Caller should pass NULL @new to * remove a policy and fall back to surrounding context--i.e. do not * install a MPOL_DEFAULT policy, nor the task or system default * mempolicy. */ int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
/* * get_policy() op must add reference [mpol_get()] to any policy at * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure * in mm/mempolicy.c will do this automatically. * get_policy() must NOT add a ref if the policy at (vma,addr) is not * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. * If no [shared/vma] mempolicy exists at the addr, get_policy() op * must return NULL--i.e., do not "fallback" to task or system default * policy. */ struct mempolicy *(*get_policy)(struct vm_area_struct *vma, unsigned long addr, pgoff_t *ilx);#endif#ifdef CONFIG_FIND_NORMAL_PAGE /* * Called by vm_normal_page() for special PTEs in @vma at @addr. This * allows for returning a "normal" page from vm_normal_page() even * though the PTE indicates that the "struct page" either does not exist * or should not be touched: "special". * * Do not add new users: this really only works when a "normal" page * was mapped, but then the PTE got changed to something weird (+ * marked special) that would not make pte_pfn() identify the originally * inserted page. */ struct page *(*find_normal_page)(struct vm_area_struct *vma, unsigned long addr);#endif /* CONFIG_FIND_NORMAL_PAGE */#ifdef CONFIG_USERFAULTFD const struct vm_uffd_ops *uffd_ops;#endif};
#ifdef CONFIG_NUMA_BALANCINGstatic inline void vma_numab_state_init(struct vm_area_struct *vma){ vma->numab_state = NULL;}static inline void vma_numab_state_free(struct vm_area_struct *vma){ kfree(vma->numab_state);}#elsestatic inline void vma_numab_state_init(struct vm_area_struct *vma) {}static inline void vma_numab_state_free(struct vm_area_struct *vma) {}#endif /* CONFIG_NUMA_BALANCING */
/* * These must be here rather than mmap_lock.h as dependent on vm_fault type, * declared in this header. */#ifdef CONFIG_PER_VMA_LOCKstatic inline void release_fault_lock(struct vm_fault *vmf){ if (vmf->flags & FAULT_FLAG_VMA_LOCK) vma_end_read(vmf->vma); else mmap_read_unlock(vmf->vma->vm_mm);}
static inline void assert_fault_locked(const struct vm_fault *vmf){ if (vmf->flags & FAULT_FLAG_VMA_LOCK) vma_assert_locked(vmf->vma); else mmap_assert_locked(vmf->vma->vm_mm);}#elsestatic inline void release_fault_lock(struct vm_fault *vmf){ mmap_read_unlock(vmf->vma->vm_mm);}
static inline void assert_fault_locked(const struct vm_fault *vmf){ mmap_assert_locked(vmf->vma->vm_mm);}#endif /* CONFIG_PER_VMA_LOCK */
static inline bool mm_flags_test(int flag, const struct mm_struct *mm){ return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline bool mm_flags_test_and_set(int flag, struct mm_struct *mm){ return test_and_set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline bool mm_flags_test_and_clear(int flag, struct mm_struct *mm){ return test_and_clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline void mm_flags_set(int flag, struct mm_struct *mm){ set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline void mm_flags_clear(int flag, struct mm_struct *mm){ clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline void mm_flags_clear_all(struct mm_struct *mm){ bitmap_zero(ACCESS_PRIVATE(&mm->flags, __mm_flags), NUM_MM_FLAG_BITS);}
extern const struct vm_operations_struct vma_dummy_vm_ops;
static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm){ memset(vma, 0, sizeof(*vma)); vma->vm_mm = mm; vma->vm_ops = &vma_dummy_vm_ops; INIT_LIST_HEAD(&vma->anon_vma_chain); vma_lock_init(vma, false);}
/* Use when VMA is not part of the VMA tree and needs no locking */static inline void vm_flags_init(struct vm_area_struct *vma, vm_flags_t flags){ VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); vma_flags_clear_all(&vma->flags); vma_flags_overwrite_word(&vma->flags, flags);}
/* * Use when VMA is part of the VMA tree and modifications need coordination * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and * it should be locked explicitly beforehand. */static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_t flags){ VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); vma_assert_write_locked(vma); vm_flags_init(vma, flags);}
static inline void vma_flags_reset_once(struct vm_area_struct *vma, vma_flags_t *flags){ const unsigned long word = flags->__vma_flags[0];
/* It is assumed only the first system word must be written once. */ vma_flags_overwrite_word_once(&vma->flags, word); /* The remainder can be copied normally. */ if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { unsigned long *dst = &vma->flags.__vma_flags[1]; const unsigned long *src = &flags->__vma_flags[1];
bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); }}
static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags){ vma_start_write(vma); vma_flags_set_word(&vma->flags, flags);}
static inline void vm_flags_clear(struct vm_area_struct *vma, vm_flags_t flags){ VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); vma_start_write(vma); vma_flags_clear_word(&vma->flags, flags);}
/* * Use only if VMA is not part of the VMA tree or has no other users and * therefore needs no locking. */static inline void __vm_flags_mod(struct vm_area_struct *vma, vm_flags_t set, vm_flags_t clear){ vm_flags_init(vma, (vma->vm_flags | set) & ~clear);}
/* * Use only when the order of set/clear operations is unimportant, otherwise * use vm_flags_{set|clear} explicitly. */static inline void vm_flags_mod(struct vm_area_struct *vma, vm_flags_t set, vm_flags_t clear){ vma_start_write(vma); __vm_flags_mod(vma, set, clear);}
static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_t bit){ const vm_flags_t mask = BIT((__force int)bit);
/* Only specific flags are permitted */ if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED))) return false;
return true;}
/* * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific * valid flags are allowed to do this. */static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit){ unsigned long *bitmap = vma->flags.__vma_flags;
vma_assert_stabilised(vma); if (__vma_atomic_valid_flag(vma, bit)) set_bit((__force int)bit, bitmap);}
/* * Test for VMA flag atomically. Requires no locks. Only specific valid flags * are allowed to do this. * * This is necessarily racey, so callers must ensure that serialisation is * achieved through some other means, or that races are permissible. */static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit){ if (__vma_atomic_valid_flag(vma, bit)) return test_bit((__force int)bit, &vma->vm_flags);
return false;}
/* Set an individual VMA flag in flags, non-atomically. */static __always_inline void vma_flags_set_flag(vma_flags_t *flags, vma_flag_t bit){ unsigned long *bitmap = flags->__vma_flags;
__set_bit((__force int)bit, bitmap);}
static __always_inline vma_flags_t __mk_vma_flags(vma_flags_t flags, size_t count, const vma_flag_t *bits){ int i;
for (i = 0; i < count; i++) vma_flags_set_flag(&flags, bits[i]); return flags;}
/* * Helper macro which bitwise-or combines the specified input flags into a * vma_flags_t bitmap value. E.g.: * * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT); * * The compiler cleverly optimises away all of the work and this ends up being * equivalent to aggregating the values manually. */#define mk_vma_flags(...) __mk_vma_flags(EMPTY_VMA_FLAGS, \ COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__})
/* * Helper macro which acts like mk_vma_flags, only appending to a copy of the * specified flags rather than establishing new flags. E.g.: * * vma_flags_t flags = append_vma_flags(VMA_STACK_DEFAULT_FLAGS, VMA_STACK_BIT, * VMA_ACCOUNT_BIT); */#define append_vma_flags(flags, ...) __mk_vma_flags(flags, \ COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__})
/* Calculates the number of set bits in the specified VMA flags. */static __always_inline int vma_flags_count(const vma_flags_t *flags){ const unsigned long *bitmap = flags->__vma_flags;
return bitmap_weight(bitmap, NUM_VMA_FLAG_BITS);}
/* * Test whether a specific VMA flag is set, e.g.: * * if (vma_flags_test(flags, VMA_READ_BIT)) { ... } */static __always_inline bool vma_flags_test(const vma_flags_t *flags, vma_flag_t bit){ const unsigned long *bitmap = flags->__vma_flags;
return test_bit((__force int)bit, bitmap);}
/* * Obtain a set of VMA flags which contain the overlapping flags contained * within flags and to_and. */static __always_inline vma_flags_t vma_flags_and_mask(const vma_flags_t *flags, vma_flags_t to_and){ vma_flags_t dst; unsigned long *bitmap_dst = dst.__vma_flags; const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_and = to_and.__vma_flags;
bitmap_and(bitmap_dst, bitmap, bitmap_to_and, NUM_VMA_FLAG_BITS); return dst;}
/* * Obtain a set of VMA flags which contains the specified overlapping flags, * e.g.: * * vma_flags_t read_flags = vma_flags_and(&flags, VMA_READ_BIT, * VMA_MAY_READ_BIT); */#define vma_flags_and(flags, ...) \ vma_flags_and_mask(flags, mk_vma_flags(__VA_ARGS__))
/* Test each of to_test flags in flags, non-atomically. */static __always_inline bool vma_flags_test_any_mask(const vma_flags_t *flags, vma_flags_t to_test){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_test = to_test.__vma_flags;
return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);}
/* * Test whether any specified VMA flag is set, e.g.: * * if (vma_flags_test_any(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */#define vma_flags_test_any(flags, ...) \ vma_flags_test_any_mask(flags, mk_vma_flags(__VA_ARGS__))
/* Test that ALL of the to_test flags are set, non-atomically. */static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags, vma_flags_t to_test){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_test = to_test.__vma_flags;
return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);}
/* * Test whether ALL specified VMA flags are set, e.g.: * * if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */#define vma_flags_test_all(flags, ...) \ vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
/* * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set * (returning false if flagmask has no flags set). * * This is defined to make the semantics clearer when testing an optionally * defined VMA flags mask, e.g.: * * if (vma_flags_test_single_mask(&flags, VMA_DROPPABLE)) { ... } * * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS * otherwise. */static __always_inline bool vma_flags_test_single_mask(const vma_flags_t *flags, vma_flags_t flagmask){ VM_WARN_ON_ONCE(vma_flags_count(&flagmask) > 1);
return vma_flags_test_any_mask(flags, flagmask);}
/* Set each of the to_set flags in flags, non-atomically. */static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set){ unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_set = to_set.__vma_flags;
bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);}
/* * Set all specified VMA flags, e.g.: * * vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); */#define vma_flags_set(flags, ...) \ vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
static __always_inline vma_flags_t __mk_vma_flags_from_masks(size_t count, const vma_flags_t *masks){ vma_flags_t flags = EMPTY_VMA_FLAGS; size_t i;
for (i = 0; i < count; i++) vma_flags_set_mask(&flags, masks[i]); return flags;}
/* * Combine pre-computed vma_flags_t masks into one value, e.g.: * * vma_flags_t flags = mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR); * * Unlike mk_vma_flags(), which takes bit numbers, this takes whole masks -- * each of which may be EMPTY_VMA_FLAGS when its feature is unavailable -- so a * bit that does not exist on the current build is never materialised. */#define mk_vma_flags_from_masks(...) \ __mk_vma_flags_from_masks(COUNT_ARGS(__VA_ARGS__), \ (const vma_flags_t []){__VA_ARGS__})
/* Clear all of the to-clear flags in flags, non-atomically. */static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear){ unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);}
/* * Clear all specified individual flags, e.g.: * * vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); */#define vma_flags_clear(flags, ...) \ vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
/* * Obtain a VMA flags value containing those flags that are present in flags or * flags_other but not in both. */static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, const vma_flags_t *flags_other){ vma_flags_t dst; const unsigned long *bitmap_other = flags_other->__vma_flags; const unsigned long *bitmap = flags->__vma_flags; unsigned long *bitmap_dst = dst.__vma_flags;
bitmap_xor(bitmap_dst, bitmap, bitmap_other, NUM_VMA_FLAG_BITS); return dst;}
/* Determine if flags and flags_other have precisely the same flags set. */static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, const vma_flags_t *flags_other){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_other = flags_other->__vma_flags;
return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS);}
/* Determine if flags and flags_other have precisely the same flags set. */static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, vma_flags_t flags_other){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_other = flags_other.__vma_flags;
return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS);}
/* * Helper macro to determine if only the specific flags are set, e.g.: * * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } */#define vma_flags_same(flags, ...) \ vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
/* * Test whether a specific flag in the VMA is set, e.g.: * * if (vma_test(vma, VMA_READ_BIT)) { ... } */static __always_inline bool vma_test(const struct vm_area_struct *vma, vma_flag_t bit){ return vma_flags_test(&vma->flags, bit);}
/* Helper to test any VMA flags in a VMA . */static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma, vma_flags_t flags){ return vma_flags_test_any_mask(&vma->flags, flags);}
/* * Helper macro for testing whether any VMA flags are set in a VMA, * e.g.: * * if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } */#define vma_test_any(vma, ...) \ vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__))
/* * Helper to test that ALL specified flags are set in a VMA. * * Note: appropriate locks must be held, this function does not acquire them for * you. */static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma, vma_flags_t flags){ return vma_flags_test_all_mask(&vma->flags, flags);}
/* * Helper macro for checking that ALL specified flags are set in a VMA, e.g.: * * if (vma_test_all(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... } */#define vma_test_all(vma, ...) \ vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__))
/* * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set * (returning false if flagmask has no flags set). * * This is useful when a flag needs to be either defined or not depending upon * kernel configuration, e.g.: * * if (vma_test_single_mask(vma, VMA_DROPPABLE)) { ... } * * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS * otherwise. */static __always_inline boolvma_test_single_mask(const struct vm_area_struct *vma, vma_flags_t flagmask){ return vma_flags_test_single_mask(&vma->flags, flagmask);}
/* * Helper to set all VMA flags in a VMA. * * Note: appropriate locks must be held, this function does not acquire them for * you. */static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma, vma_flags_t flags){ vma_flags_set_mask(&vma->flags, flags);}
/* * Helper macro for specifying VMA flags in a VMA, e.g.: * * vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); * * Note: appropriate locks must be held, this function does not acquire them for * you. */#define vma_set_flags(vma, ...) \ vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
/* Helper to clear all VMA flags in a VMA. */static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, vma_flags_t flags){ vma_flags_clear_mask(&vma->flags, flags);}
/* * Helper macro for clearing VMA flags, e.g.: * * vma_clear_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); */#define vma_clear_flags(vma, ...) \ vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
/* * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: * * if (vma_desc_test(desc, VMA_READ_BIT)) { ... } */static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, vma_flag_t bit){ return vma_flags_test(&desc->vma_flags, bit);}
/* Helper to test any VMA flags in a VMA descriptor. */static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc, vma_flags_t flags){ return vma_flags_test_any_mask(&desc->vma_flags, flags);}
/* * Helper macro for testing whether any VMA flags are set in a VMA descriptor, * e.g.: * * if (vma_desc_test_any(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } */#define vma_desc_test_any(desc, ...) \ vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__))
/* Helper to test all VMA flags in a VMA descriptor. */static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc, vma_flags_t flags){ return vma_flags_test_all_mask(&desc->vma_flags, flags);}
/* * Helper macro for testing whether ALL VMA flags are set in a VMA descriptor, * e.g.: * * if (vma_desc_test_all(desc, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */#define vma_desc_test_all(desc, ...) \ vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__))
/* Helper to set all VMA flags in a VMA descriptor. */static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, vma_flags_t flags){ vma_flags_set_mask(&desc->vma_flags, flags);}
/* * Helper macro for specifying VMA flags for an input pointer to a struct * vm_area_desc object describing a proposed VMA, e.g.: * * vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); */#define vma_desc_set_flags(desc, ...) \ vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
/* Helper to clear all VMA flags in a VMA descriptor. */static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc, vma_flags_t flags){ vma_flags_clear_mask(&desc->vma_flags, flags);}
/* * Helper macro for clearing VMA flags for an input pointer to a struct * vm_area_desc object describing a proposed VMA, e.g.: * * vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); */#define vma_desc_clear_flags(desc, ...) \ vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
static inline void vma_set_anonymous(struct vm_area_struct *vma){ vma->vm_ops = NULL;}
static inline void vma_desc_set_anonymous(struct vm_area_desc *desc){ desc->vm_ops = NULL;}
static inline bool vma_is_anonymous(struct vm_area_struct *vma){ return !vma->vm_ops;}
/* * Indicate if the VMA is a heap for the given task; for * /proc/PID/maps that is the heap of the main task. */static inline bool vma_is_initial_heap(const struct vm_area_struct *vma){ return vma->vm_start < vma->vm_mm->brk && vma->vm_end > vma->vm_mm->start_brk;}
/* * Indicate if the VMA is a stack for the given task; for * /proc/PID/maps that is the stack of the main task. */static inline bool vma_is_initial_stack(const struct vm_area_struct *vma){ /* * We make no effort to guess what a given thread considers to be * its "stack". It's not even well-defined for programs written * languages like Go. */ return vma->vm_start <= vma->vm_mm->start_stack && vma->vm_end >= vma->vm_mm->start_stack;}
static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma){ int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
if (!maybe_stack) return false;
if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == VM_STACK_INCOMPLETE_SETUP) return true;
return false;}
static inline bool vma_is_foreign(const struct vm_area_struct *vma){ if (!current->mm) return true;
if (current->mm != vma->vm_mm) return true;
return false;}
static inline bool vma_is_accessible(const struct vm_area_struct *vma){ return vma->vm_flags & VM_ACCESS_FLAGS;}
static inline bool is_shared_maywrite(const vma_flags_t *flags){ return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT);}
static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma){ return is_shared_maywrite(&vma->flags);}
/** * vma_kernel_pagesize - Default page size granularity for this VMA. * @vma: The user mapping. * * The kernel page size specifies in which granularity VMA modifications * can be performed. Folios in this VMA will be aligned to, and at least * the size of the number of bytes returned by this function. * * The default kernel page size is not affected by Transparent Huge Pages * being in effect. * * Return: The default page size granularity for this VMA. */static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma){ if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); return PAGE_SIZE;}
unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
static inlinestruct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max){ return mas_find(&vmi->mas, max - 1);}
static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi){ /* * Uses mas_find() to get the first VMA when the iterator starts. * Calling mas_next() could skip the first entry. */ return mas_find(&vmi->mas, ULONG_MAX);}
static inlinestruct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi){ return mas_next_range(&vmi->mas, ULONG_MAX);}
static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi){ return mas_prev(&vmi->mas, 0);}
static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, unsigned long start, unsigned long end, gfp_t gfp){ __mas_set_range(&vmi->mas, start, end - 1); mas_store_gfp(&vmi->mas, NULL, gfp); if (unlikely(mas_is_err(&vmi->mas))) return -ENOMEM;
return 0;}
/* Free any unused preallocations */static inline void vma_iter_free(struct vma_iterator *vmi){ mas_destroy(&vmi->mas);}
static inline int vma_iter_bulk_store(struct vma_iterator *vmi, struct vm_area_struct *vma){ vmi->mas.index = vma->vm_start; vmi->mas.last = vma->vm_end - 1; mas_store(&vmi->mas, vma); if (unlikely(mas_is_err(&vmi->mas))) return -ENOMEM;
vma_mark_attached(vma); return 0;}
static inline void vma_iter_invalidate(struct vma_iterator *vmi){ mas_pause(&vmi->mas);}
static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr){ mas_set(&vmi->mas, addr);}
#define for_each_vma(__vmi, __vma) \ while (((__vma) = vma_next(&(__vmi))) != NULL)
/* The MM code likes to work with exclusive end addresses */#define for_each_vma_range(__vmi, __vma, __end) \ while (((__vma) = vma_find(&(__vmi), (__end))) != NULL)
#ifdef CONFIG_SHMEM/* * The vma_is_shmem is not inline because it is used only by slow * paths in userfault. */bool vma_is_shmem(const struct vm_area_struct *vma);bool vma_is_anon_shmem(const struct vm_area_struct *vma);#elsestatic inline bool vma_is_shmem(const struct vm_area_struct *vma) { return false; }static inline bool vma_is_anon_shmem(const struct vm_area_struct *vma) { return false; }#endif
int vma_is_stack_for_current(const struct vm_area_struct *vma);
/* flush_tlb_range() takes a vma, not a mm, and can care about flags */#define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) }
struct mmu_gather;struct inode;
extern void prep_compound_page(struct page *page, unsigned int order);
static inline unsigned int folio_large_order(const struct folio *folio){ return folio->_flags_1 & 0xff;}
#ifdef NR_PAGES_IN_LARGE_FOLIOstatic inline unsigned long folio_large_nr_pages(const struct folio *folio){ return folio->_nr_pages;}#elsestatic inline unsigned long folio_large_nr_pages(const struct folio *folio){ return 1L << folio_large_order(folio);}#endif
/* * compound_order() can be called without holding a reference, which means * that niceties like page_folio() don't work. These callers should be * prepared to handle wild return values. For example, PG_head may be * set before the order is initialised, or this may be a tail page. * See compaction.c for some good examples. */static inline unsigned int compound_order(const struct page *page){ const struct folio *folio = (struct folio *)page;
if (!test_bit(PG_head, &folio->flags.f)) return 0; return folio_large_order(folio);}
/** * folio_order - The allocation order of a folio. * @folio: The folio. * * A folio is composed of 2^order pages. See get_order() for the definition * of order. * * Return: The order of the folio. */static inline unsigned int folio_order(const struct folio *folio){ if (!folio_test_large(folio)) return 0; return folio_large_order(folio);}
/** * folio_reset_order - Reset the folio order and derived _nr_pages * @folio: The folio. * * Reset the order and derived _nr_pages to 0. Must only be used in the * process of splitting large folios. */static inline void folio_reset_order(struct folio *folio){ if (WARN_ON_ONCE(!folio_test_large(folio))) return; folio->_flags_1 &= ~0xffUL;#ifdef NR_PAGES_IN_LARGE_FOLIO folio->_nr_pages = 0;#endif}
#include <linux/huge_mm.h>
/* * Methods to modify the page usage count. * * What counts for a page usage: * - cache mapping (page->mapping) * - private data (page->private) * - page mapped in a task's page tables, each mapping * is counted separately * * Also, many kernel routines increase the page count before a critical * routine so they can be sure the page doesn't go away from under them. */
/* * Drop a ref, return true if the refcount fell to zero (the page has no users) */static inline int put_page_testzero(struct page *page){ VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); return page_ref_dec_and_test(page);}
static inline int folio_put_testzero(struct folio *folio){ return put_page_testzero(&folio->page);}
/* * Try to grab a ref unless the page has a refcount of zero, return false if * that is the case. * This can be called when MMU is off so it must not access * any of the virtual mappings. */static inline bool get_page_unless_zero(struct page *page){ return page_ref_add_unless_zero(page, 1);}
static inline struct folio *folio_get_nontail_page(struct page *page){ if (unlikely(!get_page_unless_zero(page))) return NULL; return (struct folio *)page;}
extern int page_is_ram(unsigned long pfn);
enum { REGION_INTERSECTS, REGION_DISJOINT, REGION_MIXED,};
int region_intersects(resource_size_t offset, size_t size, unsigned long flags, unsigned long desc);
/* Support for virtually mapped pages */struct page *vmalloc_to_page(const void *addr);unsigned long vmalloc_to_pfn(const void *addr);
/* * Determine if an address is within the vmalloc range * * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there * is no special casing required. */#ifdef CONFIG_MMUextern bool is_vmalloc_addr(const void *x);extern int is_vmalloc_or_module_addr(const void *x);#elsestatic inline bool is_vmalloc_addr(const void *x){ return false;}static inline int is_vmalloc_or_module_addr(const void *x){ return 0;}#endif
/* * How many times the entire folio is mapped as a single unit (eg by a * PMD or PUD entry). This is probably not what you want, except for * debugging purposes or implementation of other core folio_*() primitives. */static inline int folio_entire_mapcount(const struct folio *folio){ VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio_large_order(folio) == 1)) return 0; return atomic_read(&folio->_entire_mapcount) + 1;}
static inline int folio_large_mapcount(const struct folio *folio){ VM_WARN_ON_FOLIO(!folio_test_large(folio), folio); return atomic_read(&folio->_large_mapcount) + 1;}
/** * folio_mapcount() - Number of mappings of this folio. * @folio: The folio. * * The folio mapcount corresponds to the number of present user page table * entries that reference any part of a folio. Each such present user page * table entry must be paired with exactly on folio reference. * * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts * exactly once. * * For hugetlb folios, each abstracted "hugetlb" user page table entry that * references the entire folio counts exactly once, even when such special * page table entries are comprised of multiple ordinary page table entries. * * Will report 0 for pages which cannot be mapped into userspace, such as * slab, page tables and similar. * * Return: The number of times this folio is mapped. */static inline int folio_mapcount(const struct folio *folio){ int mapcount;
if (likely(!folio_test_large(folio))) { mapcount = atomic_read(&folio->_mapcount) + 1; if (page_mapcount_is_type(mapcount)) mapcount = 0; return mapcount; } return folio_large_mapcount(folio);}
/** * folio_mapped - Is this folio mapped into userspace? * @folio: The folio. * * Return: True if any page in this folio is referenced by user page tables. */static inline bool folio_mapped(const struct folio *folio){ return folio_mapcount(folio) >= 1;}
static inline struct page *virt_to_head_page(const void *x){ struct page *page = virt_to_page(x);
return compound_head(page);}
static inline struct folio *virt_to_folio(const void *x){ struct page *page = virt_to_page(x);
return page_folio(page);}
void __folio_put(struct folio *folio);
void split_page(struct page *page, unsigned int order);void folio_copy(struct folio *dst, struct folio *src);int folio_mc_copy(struct folio *dst, struct folio *src);
unsigned long nr_free_buffer_pages(void);
/* Returns the number of bytes in this potentially compound page. */static inline unsigned long page_size(const struct page *page){ return PAGE_SIZE << compound_order(page);}
/* Returns the number of bits needed for the number of bytes in a page */static inline unsigned int page_shift(struct page *page){ return PAGE_SHIFT + compound_order(page);}
/** * thp_order - Order of a transparent huge page. * @page: Head page of a transparent huge page. */static inline unsigned int thp_order(struct page *page){ VM_BUG_ON_PGFLAGS(PageTail(page), page); return compound_order(page);}
/** * thp_size - Size of a transparent huge page. * @page: Head page of a transparent huge page. * * Return: Number of bytes in this page. */static inline unsigned long thp_size(struct page *page){ return PAGE_SIZE << thp_order(page);}
#ifdef CONFIG_MMU/* * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when * servicing faults for write access. In the normal case, do always want * pte_mkwrite. But get_user_pages can cause write faults for mappings * that do not have writing enabled, when used by access_process_vm. */static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma){ if (likely(vma->vm_flags & VM_WRITE)) pte = pte_mkwrite(pte, vma); return pte;}
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page);void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr);
vm_fault_t finish_fault(struct vm_fault *vmf);#endif
/* * Multiple processes may "see" the same page. E.g. for untouched * mappings of /dev/null, all processes see the same page full of * zeroes, and text pages of executables and shared libraries have * only one copy in memory, at most, normally. * * For the non-reserved pages, page_count(page) denotes a reference count. * page_count() == 0 means the page is free. page->lru is then used for * freelist management in the buddy allocator. * page_count() > 0 means the page has been allocated. * * Pages are allocated by the slab allocator in order to provide memory * to kmalloc and kmem_cache_alloc. In this case, the management of the * page, and the fields in 'struct page' are the responsibility of mm/slab.c * unless a particular usage is carefully commented. (the responsibility of * freeing the kmalloc memory is the caller's, of course). * * A page may be used by anyone else who does a __get_free_page(). * In this case, page_count still tracks the references, and should only * be used through the normal accessor functions. The top bits of page->flags * and page->virtual store page management information, but all other fields * are unused and could be used privately, carefully. The management of this * page is the responsibility of the one who allocated it, and those who have * subsequently been given references to it. * * The other pages (we may call them "pagecache pages") are completely * managed by the Linux memory manager: I/O, buffers, swapping etc. * The following discussion applies only to them. * * A pagecache page contains an opaque `private' member, which belongs to the * page's address_space. Usually, this is the address of a circular list of * the page's disk buffers. PG_private must be set to tell the VM to call * into the filesystem to release these pages. * * A folio may belong to an inode's memory mapping. In this case, * folio->mapping points to the inode, and folio->index is the file * offset of the folio, in units of PAGE_SIZE. * * If pagecache pages are not associated with an inode, they are said to be * anonymous pages. These may become associated with the swapcache, and in that * case PG_swapcache is set, and page->private is an offset into the swapcache. * * In either case (swapcache or inode backed), the pagecache itself holds one * reference to the page. Setting PG_private should also increment the * refcount. The each user mapping also has a reference to the page. * * The pagecache pages are stored in a per-mapping radix tree, which is * rooted at mapping->i_pages, and indexed by offset. * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space * lists, we instead now tag pages as dirty/writeback in the radix tree. * * All pagecache pages may be subject to I/O: * - inode pages may need to be read from disk, * - inode pages which have been modified and are MAP_SHARED may need * to be written back to the inode on disk, * - anonymous pages (including MAP_PRIVATE file mappings) which have been * modified may need to be swapped out to swap space and (later) to be read * back into memory. */
/* 127: arbitrary random number, small enough to assemble well */#define folio_ref_zero_or_close_to_overflow(folio) \ ((unsigned int) folio_ref_count(folio) + 127u <= 127u)
/** * folio_get - Increment the reference count on a folio. * @folio: The folio. * * Context: May be called in any context, as long as you know that * you have a refcount on the folio. If you do not already have one, * folio_try_get() may be the right interface for you to use. */static inline void folio_get(struct folio *folio){ VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); folio_ref_inc(folio);}
static inline void get_page(struct page *page){ struct folio *folio = page_folio(page); if (WARN_ON_ONCE(folio_test_slab(folio))) return; if (WARN_ON_ONCE(folio_test_large_kmalloc(folio))) return; folio_get(folio);}
static inline __must_check bool try_get_page(struct page *page){ page = compound_head(page); if (WARN_ON_ONCE(page_ref_count(page) <= 0)) return false; page_ref_inc(page); return true;}
/** * folio_put - Decrement the reference count on a folio. * @folio: The folio. * * If the folio's reference count reaches zero, the memory will be * released back to the page allocator and may be used by another * allocation immediately. Do not access the memory or the struct folio * after calling folio_put() unless you can be sure that it wasn't the * last reference. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */static inline void folio_put(struct folio *folio){ if (folio_put_testzero(folio)) __folio_put(folio);}
/** * folio_put_refs - Reduce the reference count on a folio. * @folio: The folio. * @refs: The amount to subtract from the folio's reference count. * * If the folio's reference count reaches zero, the memory will be * released back to the page allocator and may be used by another * allocation immediately. Do not access the memory or the struct folio * after calling folio_put_refs() unless you can be sure that these weren't * the last references. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */static inline void folio_put_refs(struct folio *folio, int refs){ if (folio_ref_sub_and_test(folio, refs)) __folio_put(folio);}
void folios_put_refs(struct folio_batch *folios, unsigned int *refs);
/* * union release_pages_arg - an array of pages or folios * * release_pages() releases a simple array of multiple pages, and * accepts various different forms of said page array: either * a regular old boring array of pages, an array of folios, or * an array of encoded page pointers. * * The transparent union syntax for this kind of "any of these * argument types" is all kinds of ugly, so look away. */typedef union { struct page **pages; struct folio **folios; struct encoded_page **encoded_pages;} release_pages_arg __attribute__ ((__transparent_union__));
void release_pages(release_pages_arg, int nr);
/** * folios_put - Decrement the reference count on an array of folios. * @folios: The folios. * * Like folio_put(), but for a batch of folios. This is more efficient * than writing the loop yourself as it will optimise the locks which need * to be taken if the folios are freed. The folios batch is returned * empty and ready to be reused for another batch; there is no need to * reinitialise it. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */static inline void folios_put(struct folio_batch *folios){ folios_put_refs(folios, NULL);}
static inline void put_page(struct page *page){ struct folio *folio = page_folio(page);
if (folio_test_slab(folio) || folio_test_large_kmalloc(folio)) return;
folio_put(folio);}
/* * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload * the page's refcount so that two separate items are tracked: the original page * reference count, and also a new count of how many pin_user_pages() calls were * made against the page. ("gup-pinned" is another term for the latter). * * With this scheme, pin_user_pages() becomes special: such pages are marked as * distinct from normal pages. As such, the unpin_user_page() call (and its * variants) must be used in order to release gup-pinned pages. * * Choice of value: * * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference * counts with respect to pin_user_pages() and unpin_user_page() becomes * simpler, due to the fact that adding an even power of two to the page * refcount has the effect of using only the upper N bits, for the code that * counts up using the bias value. This means that the lower bits are left for * the exclusive use of the original code that increments and decrements by one * (or at least, by much smaller values than the bias value). * * Of course, once the lower bits overflow into the upper bits (and this is * OK, because subtraction recovers the original values), then visual inspection * no longer suffices to directly view the separate counts. However, for normal * applications that don't have huge page reference counts, this won't be an * issue. * * Locking: the lockless algorithm described in folio_try_get_rcu() * provides safe operation for get_user_pages(), folio_mkclean() and * other calls that race to set up page table entries. */#define GUP_PIN_COUNTING_BIAS (1U << 10)
void unpin_user_page(struct page *page);void unpin_folio(struct folio *folio);void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, bool make_dirty);void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, bool make_dirty);void unpin_user_pages(struct page **pages, unsigned long npages);void unpin_user_folio(struct folio *folio, unsigned long npages);void unpin_folios(struct folio **folios, unsigned long nfolios);
static inline bool is_cow_mapping(vm_flags_t flags){ return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;}
static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc){ const vma_flags_t *flags = &desc->vma_flags;
return vma_flags_test(flags, VMA_MAYWRITE_BIT) && !vma_flags_test(flags, VMA_SHARED_BIT);}
#ifndef CONFIG_MMUstatic inline bool is_nommu_shared_mapping(vm_flags_t flags){ /* * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of * a file mapping. R/O MAP_PRIVATE mappings might still modify * underlying memory if ptrace is active, so this is only possible if * ptrace does not apply. Note that there is no mprotect() to upgrade * write permissions later. */ return flags & (VM_MAYSHARE | VM_MAYOVERLAY);}
static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags){ return vma_flags_test_any(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT);}#endif
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)#define SECTION_IN_PAGE_FLAGS#endif
/* * The identification function is mainly used by the buddy allocator for * determining if two pages could be buddies. We are not really identifying * the zone since we could be using the section number id if we do not have * node id available in page flags. * We only guarantee that it will return the same value for two combinable * pages in a zone. */static inline int page_zone_id(struct page *page){ return (page->flags.f >> ZONEID_PGSHIFT) & ZONEID_MASK;}
#ifdef NODE_NOT_IN_PAGE_FLAGSint memdesc_nid(memdesc_flags_t mdf);#elsestatic inline int memdesc_nid(memdesc_flags_t mdf){ return (mdf.f >> NODES_PGSHIFT) & NODES_MASK;}#endif
static inline int page_to_nid(const struct page *page){ return memdesc_nid(PF_POISONED_CHECK(page)->flags);}
static inline int folio_nid(const struct folio *folio){ return memdesc_nid(folio->flags);}
#ifdef CONFIG_NUMA_BALANCING/* page access time bits needs to hold at least 4 seconds */#define PAGE_ACCESS_TIME_MIN_BITS 12#if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS#define PAGE_ACCESS_TIME_BUCKETS \ (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT)#else#define PAGE_ACCESS_TIME_BUCKETS 0#endif
#define PAGE_ACCESS_TIME_MASK \ (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS)
static inline int cpu_pid_to_cpupid(int cpu, int pid){ return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);}
static inline int cpupid_to_pid(int cpupid){ return cpupid & LAST__PID_MASK;}
static inline int cpupid_to_cpu(int cpupid){ return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;}
static inline int cpupid_to_nid(int cpupid){ return cpu_to_node(cpupid_to_cpu(cpupid));}
static inline bool cpupid_pid_unset(int cpupid){ return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);}
static inline bool cpupid_cpu_unset(int cpupid){ return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);}
static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid){ return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);}
#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGSstatic inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid){ return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK);}
static inline int folio_last_cpupid(struct folio *folio){ return folio->_last_cpupid;}static inline void page_cpupid_reset_last(struct page *page){ page->_last_cpupid = -1 & LAST_CPUPID_MASK;}#elsestatic inline int folio_last_cpupid(struct folio *folio){ return (folio->flags.f >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;}
int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
static inline void page_cpupid_reset_last(struct page *page){ page->flags.f |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;}#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
static inline int folio_xchg_access_time(struct folio *folio, int time){ int last_time;
last_time = folio_xchg_last_cpupid(folio, time >> PAGE_ACCESS_TIME_BUCKETS); return last_time << PAGE_ACCESS_TIME_BUCKETS;}
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma){ unsigned int pid_bit;
pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) { __set_bit(pid_bit, &vma->numab_state->pids_active[1]); }}
bool folio_use_access_time(struct folio *folio);#else /* !CONFIG_NUMA_BALANCING */static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid){ return folio_nid(folio); /* XXX */}
static inline int folio_xchg_access_time(struct folio *folio, int time){ return 0;}
static inline int folio_last_cpupid(struct folio *folio){ return folio_nid(folio); /* XXX */}
static inline int cpupid_to_nid(int cpupid){ return -1;}
static inline int cpupid_to_pid(int cpupid){ return -1;}
static inline int cpupid_to_cpu(int cpupid){ return -1;}
static inline int cpu_pid_to_cpupid(int nid, int pid){ return -1;}
static inline bool cpupid_pid_unset(int cpupid){ return true;}
static inline void page_cpupid_reset_last(struct page *page){}
static inline bool cpupid_match_pid(struct task_struct *task, int cpupid){ return false;}
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma){}static inline bool folio_use_access_time(struct folio *folio){ return false;}#endif /* CONFIG_NUMA_BALANCING */
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
/* * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid * setting tags for all pages to native kernel tag value 0xff, as the default * value 0x00 maps to 0xff. */
static inline u8 page_kasan_tag(const struct page *page){ u8 tag = KASAN_TAG_KERNEL;
if (kasan_enabled()) { tag = (page->flags.f >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; tag ^= 0xff; }
return tag;}
static inline void page_kasan_tag_set(struct page *page, u8 tag){ unsigned long old_flags, flags;
if (!kasan_enabled()) return;
tag ^= 0xff; old_flags = READ_ONCE(page->flags.f); do { flags = old_flags; flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags)));}
static inline void page_kasan_tag_reset(struct page *page){ if (kasan_enabled()) page_kasan_tag_set(page, KASAN_TAG_KERNEL);}
#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline u8 page_kasan_tag(const struct page *page){ return 0xff;}
static inline void page_kasan_tag_set(struct page *page, u8 tag) { }static inline void page_kasan_tag_reset(struct page *page) { }
#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline struct zone *page_zone(const struct page *page){ return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];}
static inline pg_data_t *page_pgdat(const struct page *page){ return NODE_DATA(page_to_nid(page));}
static inline pg_data_t *folio_pgdat(const struct folio *folio){ return NODE_DATA(folio_nid(folio));}
static inline struct zone *folio_zone(const struct folio *folio){ return &folio_pgdat(folio)->node_zones[folio_zonenum(folio)];}
#ifdef SECTION_IN_PAGE_FLAGSstatic inline void set_page_section(struct page *page, unsigned long section){ page->flags.f &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;}
static inline unsigned long memdesc_section(memdesc_flags_t mdf){ return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;}#else /* !SECTION_IN_PAGE_FLAGS */static inline unsigned long memdesc_section(memdesc_flags_t mdf){ return 0;}#endif /* SECTION_IN_PAGE_FLAGS */
/** * folio_pfn - Return the Page Frame Number of a folio. * @folio: The folio. * * A folio may contain multiple pages. The pages have consecutive * Page Frame Numbers. * * Return: The Page Frame Number of the first page in the folio. */static inline unsigned long folio_pfn(const struct folio *folio){ return page_to_pfn(&folio->page);}
static inline struct folio *pfn_folio(unsigned long pfn){ return page_folio(pfn_to_page(pfn));}
#ifdef CONFIG_MMUstatic inline pte_t mk_pte(const struct page *page, pgprot_t pgprot){ return pfn_pte(page_to_pfn(page), pgprot);}
/** * folio_mk_pte - Create a PTE for this folio * @folio: The folio to create a PTE for * @pgprot: The page protection bits to use * * Create a page table entry for the first page of this folio. * This is suitable for passing to set_ptes(). * * Return: A page table entry suitable for mapping this folio. */static inline pte_t folio_mk_pte(const struct folio *folio, pgprot_t pgprot){ return pfn_pte(folio_pfn(folio), pgprot);}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE/** * folio_mk_pmd - Create a PMD for this folio * @folio: The folio to create a PMD for * @pgprot: The page protection bits to use * * Create a page table entry for the first page of this folio. * This is suitable for passing to set_pmd_at(). * * Return: A page table entry suitable for mapping this folio. */static inline pmd_t folio_mk_pmd(const struct folio *folio, pgprot_t pgprot){ return pmd_mkhuge(pfn_pmd(folio_pfn(folio), pgprot));}
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD/** * folio_mk_pud - Create a PUD for this folio * @folio: The folio to create a PUD for * @pgprot: The page protection bits to use * * Create a page table entry for the first page of this folio. * This is suitable for passing to set_pud_at(). * * Return: A page table entry suitable for mapping this folio. */static inline pud_t folio_mk_pud(const struct folio *folio, pgprot_t pgprot){ return pud_mkhuge(pfn_pud(folio_pfn(folio), pgprot));}#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif /* CONFIG_MMU */
static inline bool folio_has_pincount(const struct folio *folio){ if (IS_ENABLED(CONFIG_64BIT)) return folio_test_large(folio); return folio_order(folio) > 1;}
/** * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. * @folio: The folio. * * This function checks if a folio has been pinned via a call to * a function in the pin_user_pages() family. * * For small folios, the return value is partially fuzzy: false is not fuzzy, * because it means "definitely not pinned for DMA", but true means "probably * pinned for DMA, but possibly a false positive due to having at least * GUP_PIN_COUNTING_BIAS worth of normal folio references". * * False positives are OK, because: a) it's unlikely for a folio to * get that many refcounts, and b) all the callers of this routine are * expected to be able to deal gracefully with a false positive. * * For most large folios, the result will be exactly correct. That's because * we have more tracking data available: the _pincount field is used * instead of the GUP_PIN_COUNTING_BIAS scheme. * * For more information, please see Documentation/core-api/pin_user_pages.rst. * * Return: True, if it is likely that the folio has been "dma-pinned". * False, if the folio is definitely not dma-pinned. */static inline bool folio_maybe_dma_pinned(struct folio *folio){ if (folio_has_pincount(folio)) return atomic_read(&folio->_pincount) > 0;
/* * folio_ref_count() is signed. If that refcount overflows, then * folio_ref_count() returns a negative value, and callers will avoid * further incrementing the refcount. * * Here, for that overflow case, use the sign bit to count a little * bit higher via unsigned math, and thus still get an accurate result. */ return ((unsigned int)folio_ref_count(folio)) >= GUP_PIN_COUNTING_BIAS;}
/* * This should most likely only be called during fork() to see whether we * should break the cow immediately for an anon page on the src mm. * * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. */static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma, struct folio *folio){ VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1));
if (!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)) return false;
return folio_maybe_dma_pinned(folio);}
/** * is_zero_page - Query if a page is a zero page * @page: The page to query * * This returns true if @page is one of the permanent zero pages. */static inline bool is_zero_page(const struct page *page){ return is_zero_pfn(page_to_pfn(page));}
/** * is_zero_folio - Query if a folio is a zero page * @folio: The folio to query * * This returns true if @folio is one of the permanent zero pages. */static inline bool is_zero_folio(const struct folio *folio){ return is_zero_page(&folio->page);}
/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */#ifdef CONFIG_MIGRATIONstatic inline bool folio_is_longterm_pinnable(struct folio *folio){#ifdef CONFIG_CMA int mt = folio_migratetype(folio);
if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) return false;#endif /* The zero page can be "pinned" but gets special handling. */ if (is_zero_folio(folio)) return true;
/* Coherent device memory must always allow eviction. */ if (folio_is_device_coherent(folio)) return false;
/* * Filesystems can only tolerate transient delays to truncate and * hole-punch operations */ if (folio_is_fsdax(folio)) return false;
/* Otherwise, non-movable zone folios can be pinned. */ return !folio_is_zone_movable(folio);
}#elsestatic inline bool folio_is_longterm_pinnable(struct folio *folio){ return true;}#endif
static inline void set_page_zone(struct page *page, enum zone_type zone){ page->flags.f &= ~(ZONES_MASK << ZONES_PGSHIFT); page->flags.f |= (zone & ZONES_MASK) << ZONES_PGSHIFT;}
static inline void set_page_node(struct page *page, unsigned long node){ page->flags.f &= ~(NODES_MASK << NODES_PGSHIFT); page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT;}
static inline void set_page_links(struct page *page, enum zone_type zone, unsigned long node, unsigned long pfn){ set_page_zone(page, zone); set_page_node(page, node);#ifdef SECTION_IN_PAGE_FLAGS set_page_section(page, pfn_to_section_nr(pfn));#endif}
/** * folio_nr_pages - The number of pages in the folio. * @folio: The folio. * * Return: A positive power of two. */static inline unsigned long folio_nr_pages(const struct folio *folio){ if (!folio_test_large(folio)) return 1; return folio_large_nr_pages(folio);}
/* * compound_nr() returns the number of pages in this potentially compound * page. compound_nr() can be called on a tail page, and is defined to * return 1 in that case. */static inline unsigned long compound_nr(const struct page *page){ const struct folio *folio = (struct folio *)page;
if (!test_bit(PG_head, &folio->flags.f)) return 1; return folio_large_nr_pages(folio);}
/** * folio_next - Move to the next physical folio. * @folio: The folio we're currently operating on. * * If you have physically contiguous memory which may span more than * one folio (eg a &struct bio_vec), use this function to move from one * folio to the next. Do not use it if the memory is only virtually * contiguous as the folios are almost certainly not adjacent to each * other. This is the folio equivalent to writing ``page++``. * * Context: We assume that the folios are refcounted and/or locked at a * higher level and do not adjust the reference counts. * Return: The next struct folio. */static inline struct folio *folio_next(struct folio *folio){ return (struct folio *)folio_page(folio, folio_nr_pages(folio));}
/** * folio_shift - The size of the memory described by this folio. * @folio: The folio. * * A folio represents a number of bytes which is a power-of-two in size. * This function tells you which power-of-two the folio is. See also * folio_size() and folio_order(). * * Context: The caller should have a reference on the folio to prevent * it from being split. It is not necessary for the folio to be locked. * Return: The base-2 logarithm of the size of this folio. */static inline unsigned int folio_shift(const struct folio *folio){ return PAGE_SHIFT + folio_order(folio);}
/** * folio_size - The number of bytes in a folio. * @folio: The folio. * * Context: The caller should have a reference on the folio to prevent * it from being split. It is not necessary for the folio to be locked. * Return: The number of bytes in this folio. */static inline size_t folio_size(const struct folio *folio){ return PAGE_SIZE << folio_order(folio);}
/** * folio_maybe_mapped_shared - Whether the folio is mapped into the page * tables of more than one MM * @folio: The folio. * * This function checks if the folio maybe currently mapped into more than one * MM ("maybe mapped shared"), or if the folio is certainly mapped into a single * MM ("mapped exclusively"). * * For KSM folios, this function also returns "mapped shared" when a folio is * mapped multiple times into the same MM, because the individual page mappings * are independent. * * For small anonymous folios and anonymous hugetlb folios, the return * value will be exactly correct: non-KSM folios can only be mapped at most once * into an MM, and they cannot be partially mapped. KSM folios are * considered shared even if mapped multiple times into the same MM. * * For other folios, the result can be fuzzy: * #. For partially-mappable large folios (THP), the return value can wrongly * indicate "mapped shared" (false positive) if a folio was mapped by * more than two MMs at one point in time. * #. For pagecache folios (including hugetlb), the return value can wrongly * indicate "mapped shared" (false positive) when two VMAs in the same MM * cover the same file range. * * Further, this function only considers current page table mappings that * are tracked using the folio mapcount(s). * * This function does not consider: * #. If the folio might get mapped in the (near) future (e.g., swapcache, * pagecache, temporary unmapping for migration). * #. If the folio is mapped differently (VM_PFNMAP). * #. If hugetlb page table sharing applies. Callers might want to check * hugetlb_pmd_shared(). * * Return: Whether the folio is estimated to be mapped into more than one MM. */static inline bool folio_maybe_mapped_shared(struct folio *folio){ int mapcount = folio_mapcount(folio);
/* Only partially-mappable folios require more care. */ if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio))) return mapcount > 1;
/* * vm_insert_page() without CONFIG_TRANSPARENT_HUGEPAGE ... * simply assume "mapped shared", nobody should really care * about this for arbitrary kernel allocations. */ if (!IS_ENABLED(CONFIG_MM_ID)) return true;
/* * A single mapping implies "mapped exclusively", even if the * folio flag says something different: it's easier to handle this * case here instead of on the RMAP hot path. */ if (mapcount <= 1) return false; return test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids);}
/** * folio_expected_ref_count - calculate the expected folio refcount * @folio: the folio * * Calculate the expected folio refcount, taking references from the pagecache, * swapcache, PG_private and page table mappings into account. Useful in * combination with folio_ref_count() to detect unexpected references (e.g., * GUP or other temporary references). * * Does currently not consider references from the LRU cache. If the folio * was isolated from the LRU (which is the case during migration or split), * the LRU cache does not apply. * * Calling this function on an unmapped folio -- !folio_mapped() -- that is * locked will return a stable result. * * Calling this function on a mapped folio will not result in a stable result, * because nothing stops additional page table mappings from coming (e.g., * fork()) or going (e.g., munmap()). * * Calling this function without the folio lock will also not result in a * stable result: for example, the folio might get dropped from the swapcache * concurrently. * * However, even when called without the folio lock or on a mapped folio, * this function can be used to detect unexpected references early (for example, * if it makes sense to even lock the folio and unmap it). * * The caller must add any reference (e.g., from folio_try_get()) it might be * holding itself to the result. * * Returns: the expected folio refcount. */static inline int folio_expected_ref_count(const struct folio *folio){ const int order = folio_order(folio); int ref_count = 0;
if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio))) return 0;
/* One reference per page from the swapcache. */ ref_count += folio_test_swapcache(folio) << order;
if (!folio_test_anon(folio)) { /* One reference per page from the pagecache. */ ref_count += !!folio->mapping << order; /* One reference from PG_private. */ ref_count += folio_test_private(folio); }
/* One reference per page table mapping. */ return ref_count + folio_mapcount(folio);}
#ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLEstatic inline int arch_make_folio_accessible(struct folio *folio){ return 0;}#endif
/* * Some inline functions in vmstat.h depend on page_zone() */#include <linux/vmstat.h>
#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)#define HASHED_PAGE_VIRTUAL#endif
#if defined(WANT_PAGE_VIRTUAL)static inline void *page_address(const struct page *page){ return page->virtual;}static inline void set_page_address(struct page *page, void *address){ page->virtual = address;}#define page_address_init() do { } while(0)#endif
#if defined(HASHED_PAGE_VIRTUAL)void *page_address(const struct page *page);void set_page_address(struct page *page, void *virtual);void page_address_init(void);#endif
static __always_inline void *lowmem_page_address(const struct page *page){ return page_to_virt(page);}
#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)#define page_address(page) lowmem_page_address(page)#define set_page_address(page, address) do { } while(0)#define page_address_init() do { } while(0)#endif
static inline void *folio_address(const struct folio *folio){ return page_address(&folio->page);}
/* * Return true only if the page has been allocated with * ALLOC_NO_WATERMARKS and the low watermark was not * met implying that the system is under some pressure. */static inline bool page_is_pfmemalloc(const struct page *page){ /* * lru.next has bit 1 set if the page is allocated from the * pfmemalloc reserves. Callers may simply overwrite it if * they do not need to preserve that information. */ return (uintptr_t)page->lru.next & BIT(1);}
/* * Return true only if the folio has been allocated with * ALLOC_NO_WATERMARKS and the low watermark was not * met implying that the system is under some pressure. */static inline bool folio_is_pfmemalloc(const struct folio *folio){ /* * lru.next has bit 1 set if the page is allocated from the * pfmemalloc reserves. Callers may simply overwrite it if * they do not need to preserve that information. */ return (uintptr_t)folio->lru.next & BIT(1);}
/* * Only to be called by the page allocator on a freshly allocated * page. */static inline void set_page_pfmemalloc(struct page *page){ page->lru.next = (void *)BIT(1);}
static inline void clear_page_pfmemalloc(struct page *page){ page->lru.next = NULL;}
/* * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. */extern void pagefault_out_of_memory(void);
#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)#define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1))
/* * Parameter block passed down to zap_pte_range in exceptional cases. */struct zap_details { struct folio *single_folio; /* Locked folio to be unmapped */ bool skip_cows; /* Do not zap COWed private pages */ bool reclaim_pt; /* Need reclaim page tables? */ bool reaping; /* Reaping, do not block. */ zap_flags_t zap_flags; /* Extra flags for zapping */};
/* * Whether to drop the pte markers, for example, the uffd-wp information for * file-backed memory. This should only be specified when we will completely * drop the page in the mm, either by truncation or unmapping of the vma. By * default, the flag is not set. */#define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0))/* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */#define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1))
#ifdef CONFIG_MMUextern bool can_do_mlock(void);#elsestatic inline bool can_do_mlock(void) { return false; }#endifextern int user_shm_lock(size_t, struct ucounts *);extern void user_shm_unlock(size_t, struct ucounts *);
struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, pte_t pte);struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte);struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd);struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd);struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud);
void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size);void zap_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size);/** * zap_vma - zap all page table entries in a vma * @vma: The vma to zap. */static inline void zap_vma(struct vm_area_struct *vma){ zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start);}struct mmu_notifier_range;
void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling);intcopy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write);
struct follow_pfnmap_args { /** * Inputs: * @vma: Pointer to @vm_area_struct struct * @address: the virtual address to walk */ struct vm_area_struct *vma; unsigned long address; /** * Internals: * * The caller shouldn't touch any of these. */ spinlock_t *lock; pte_t *ptep; /** * Outputs: * * @pfn: the PFN of the address * @addr_mask: address mask covering pfn * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; unsigned long addr_mask; pgprot_t pgprot; bool writable; bool special;};int follow_pfnmap_start(struct follow_pfnmap_args *args);void follow_pfnmap_end(struct follow_pfnmap_args *args);
extern void truncate_pagecache(struct inode *inode, loff_t new);extern void truncate_setsize(struct inode *inode, loff_t newsize);void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);int generic_error_remove_folio(struct address_space *mapping, struct folio *folio);
struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, unsigned long address, struct pt_regs *regs);
#ifdef CONFIG_MMUextern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags, struct pt_regs *regs);extern int fixup_user_fault(struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked);void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows);void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows);#elsestatic inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags, struct pt_regs *regs){ /* should never happen if there's no MMU */ BUG(); return VM_FAULT_SIGBUS;}static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked){ /* should never happen if there's no MMU */ BUG(); return -EFAULT;}static inline void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows) { }static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { }#endif
static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen){ unmap_mapping_range(mapping, holebegin, holelen, 0);}
static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr);
extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags);extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags);
#ifdef CONFIG_BPF_SYSCALLextern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags);#endif
long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked);long pin_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked);
/* * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. */static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, unsigned long addr, int gup_flags, struct vm_area_struct **vmap){ struct page *page; struct vm_area_struct *vma; int got;
if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT))) return ERR_PTR(-EINVAL);
got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL);
if (got < 0) return ERR_PTR(got);
vma = vma_lookup(mm, addr); if (WARN_ON_ONCE(!vma)) { put_page(page); return ERR_PTR(-EINVAL); }
*vmap = vma; return page;}
long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages);long pin_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages);long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, struct page **pages, unsigned int gup_flags);long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, struct page **pages, unsigned int gup_flags);long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, struct folio **folios, unsigned int max_folios, pgoff_t *offset);int folio_add_pins(struct folio *folio, unsigned int pins);
int get_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages);int pin_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages);void folio_add_pin(struct folio *folio);
int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim);
struct kvec;struct page *get_dump_page(unsigned long addr, int *locked);
bool folio_mark_dirty(struct folio *folio);bool folio_mark_dirty_lock(struct folio *folio);bool set_page_dirty(struct page *page);int set_page_dirty_lock(struct page *page);
int get_cmdline(struct task_struct *task, char *buffer, int buflen);
/* * Flags used by change_protection(). For now we make it a bitmap so * that we can pass in multiple flags just like parameters. However * for now all the callers are only use one of the flags at the same * time. *//* * Whether we should manually check if we can map individual PTEs writable, * because something (e.g., COW, uffd-wp) blocks that from happening for all * PTEs automatically in a writable mapping. */#define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0)/* Whether this protection change is for NUMA hints */#define MM_CP_PROT_NUMA (1UL << 1)/* Whether this change is for write protecting */#define MM_CP_UFFD_WP (1UL << 2) /* do wp */#define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE)
bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte);extern long change_protection(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long cp_flags);extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, vm_flags_t newflags);
/* * doesn't attempt to fault and will return short. */int get_user_pages_fast_only(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages);
static inline bool get_user_page_fast_only(unsigned long addr, unsigned int gup_flags, struct page **pagep){ return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;}/* * per-process(per-mm_struct) statistics. */static inline unsigned long get_mm_counter(struct mm_struct *mm, int member){ return percpu_counter_read_positive(&mm->rss_stat[member]);}
static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member){ return percpu_counter_sum_positive(&mm->rss_stat[member]);}
void mm_trace_rss_stat(struct mm_struct *mm, int member);
static inline void add_mm_counter(struct mm_struct *mm, int member, long value){ percpu_counter_add(&mm->rss_stat[member], value);
mm_trace_rss_stat(mm, member);}
static inline void inc_mm_counter(struct mm_struct *mm, int member){ percpu_counter_inc(&mm->rss_stat[member]);
mm_trace_rss_stat(mm, member);}
static inline void dec_mm_counter(struct mm_struct *mm, int member){ percpu_counter_dec(&mm->rss_stat[member]);
mm_trace_rss_stat(mm, member);}
/* Optimized variant when folio is already known not to be anon */static inline int mm_counter_file(struct folio *folio){ if (folio_test_swapbacked(folio)) return MM_SHMEMPAGES; return MM_FILEPAGES;}
static inline int mm_counter(struct folio *folio){ if (folio_test_anon(folio)) return MM_ANONPAGES; return mm_counter_file(folio);}
static inline unsigned long get_mm_rss(struct mm_struct *mm){ return get_mm_counter(mm, MM_FILEPAGES) + get_mm_counter(mm, MM_ANONPAGES) + get_mm_counter(mm, MM_SHMEMPAGES);}
static inline unsigned long get_mm_rss_sum(struct mm_struct *mm){ return get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_ANONPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES);}
static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm){ return max(mm->hiwater_rss, get_mm_rss(mm));}
static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm){ return max(mm->hiwater_vm, mm->total_vm);}
static inline void update_hiwater_rss(struct mm_struct *mm){ unsigned long _rss = get_mm_rss(mm);
if (data_race(mm->hiwater_rss) < _rss) data_race(mm->hiwater_rss = _rss);}
static inline void update_hiwater_vm(struct mm_struct *mm){ if (mm->hiwater_vm < mm->total_vm) mm->hiwater_vm = mm->total_vm;}
static inline void reset_mm_hiwater_rss(struct mm_struct *mm){ mm->hiwater_rss = get_mm_rss(mm);}
static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm){ unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
if (*maxrss < hiwater_rss) *maxrss = hiwater_rss;}
#ifndef CONFIG_ARCH_HAS_PTE_SPECIALstatic inline int pte_special(pte_t pte){ return 0;}
static inline pte_t pte_mkspecial(pte_t pte){ return pte;}#endif
#ifndef CONFIG_ARCH_SUPPORTS_PMD_PFNMAPstatic inline bool pmd_special(pmd_t pmd){ return false;}
static inline pmd_t pmd_mkspecial(pmd_t pmd){ return pmd;}#endif /* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */
#ifndef CONFIG_ARCH_SUPPORTS_PUD_PFNMAPstatic inline bool pud_special(pud_t pud){ return false;}
static inline pud_t pud_mkspecial(pud_t pud){ return pud;}#endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
#ifdef __PAGETABLE_P4D_FOLDEDstatic inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address){ return 0;}#elseint __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);#endif
#if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address){ return 0;}static inline void mm_inc_nr_puds(struct mm_struct *mm) {}static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
#elseint __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
static inline void mm_inc_nr_puds(struct mm_struct *mm){ if (mm_pud_folded(mm)) return; atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);}
static inline void mm_dec_nr_puds(struct mm_struct *mm){ if (mm_pud_folded(mm)) return; atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);}#endif
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address){ return 0;}
static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
#elseint __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
static inline void mm_inc_nr_pmds(struct mm_struct *mm){ if (mm_pmd_folded(mm)) return; atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);}
static inline void mm_dec_nr_pmds(struct mm_struct *mm){ if (mm_pmd_folded(mm)) return; atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);}#endif
#ifdef CONFIG_MMUstatic inline void mm_pgtables_bytes_init(struct mm_struct *mm){ atomic_long_set(&mm->pgtables_bytes, 0);}
static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm){ return atomic_long_read(&mm->pgtables_bytes);}
static inline void mm_inc_nr_ptes(struct mm_struct *mm){ atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);}
static inline void mm_dec_nr_ptes(struct mm_struct *mm){ atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);}#else
static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm){ return 0;}
static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}#endif
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);int __pte_alloc_kernel(pmd_t *pmd);
#if defined(CONFIG_MMU)
static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address){ return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? NULL : p4d_offset(pgd, address);}
static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address){ return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? NULL : pud_offset(p4d, address);}
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address){ return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? NULL: pmd_offset(pud, address);}#endif /* CONFIG_MMU */
enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, /* High bits are used for zone/node/section */};
static inline struct ptdesc *virt_to_ptdesc(const void *x){ return page_ptdesc(virt_to_page(x));}
/** * ptdesc_address - Virtual address of page table. * @pt: Page table descriptor. * * Return: The first byte of the page table described by @pt. */static inline void *ptdesc_address(const struct ptdesc *pt){ return folio_address(ptdesc_folio(pt));}
static inline bool pagetable_is_reserved(struct ptdesc *pt){ return test_bit(PT_reserved, &pt->pt_flags.f);}
/** * ptdesc_set_kernel - Mark a ptdesc used to map the kernel * @ptdesc: The ptdesc to be marked * * Kernel page tables often need special handling. Set a flag so that * the handling code knows this ptdesc will not be used for userspace. */static inline void ptdesc_set_kernel(struct ptdesc *ptdesc){ set_bit(PT_kernel, &ptdesc->pt_flags.f);}
/** * ptdesc_clear_kernel - Mark a ptdesc as no longer used to map the kernel * @ptdesc: The ptdesc to be unmarked * * Use when the ptdesc is no longer used to map the kernel and no longer * needs special handling. */static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc){ /* * Note: the 'PG_referenced' bit does not strictly need to be * cleared before freeing the page. But this is nice for * symmetry. */ clear_bit(PT_kernel, &ptdesc->pt_flags.f);}
/** * ptdesc_test_kernel - Check if a ptdesc is used to map the kernel * @ptdesc: The ptdesc being tested * * Call to tell if the ptdesc used to map the kernel. */static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc){ return test_bit(PT_kernel, &ptdesc->pt_flags.f);}
/** * pagetable_alloc - Allocate pagetables * @gfp: GFP flags * @order: desired pagetable order * * pagetable_alloc allocates memory for page tables as well as a page table * descriptor to describe that memory. * * Return: The ptdesc describing the allocated page tables. */static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order){ struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order);
return page_ptdesc(page);}#define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__))
static inline void __pagetable_free(struct ptdesc *pt){ struct page *page = ptdesc_page(pt);
__free_pages(page, compound_order(page));}
#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREEvoid pagetable_free_kernel(struct ptdesc *pt);#elsestatic inline void pagetable_free_kernel(struct ptdesc *pt){ __pagetable_free(pt);}#endif/** * pagetable_free - Free pagetables * @pt: The page table descriptor * * pagetable_free frees the memory of all page tables described by a page * table descriptor and the memory for the descriptor itself. */static inline void pagetable_free(struct ptdesc *pt){ if (ptdesc_test_kernel(pt)) { ptdesc_clear_kernel(pt); pagetable_free_kernel(pt); } else { __pagetable_free(pt); }}
#if defined(CONFIG_SPLIT_PTE_PTLOCKS)#if ALLOC_SPLIT_PTLOCKSvoid __init ptlock_cache_init(void);bool ptlock_alloc(struct ptdesc *ptdesc);void ptlock_free(struct ptdesc *ptdesc);
static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc){ return ptdesc->ptl;}#else /* ALLOC_SPLIT_PTLOCKS */static inline void ptlock_cache_init(void){}
static inline bool ptlock_alloc(struct ptdesc *ptdesc){ return true;}
static inline void ptlock_free(struct ptdesc *ptdesc){}
static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc){ return &ptdesc->ptl;}#endif /* ALLOC_SPLIT_PTLOCKS */
static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd){ return ptlock_ptr(page_ptdesc(pmd_page(*pmd)));}
static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte){ BUILD_BUG_ON(IS_ENABLED(CONFIG_HIGHPTE)); BUILD_BUG_ON(MAX_PTRS_PER_PTE * sizeof(pte_t) > PAGE_SIZE); return ptlock_ptr(virt_to_ptdesc(pte));}
static inline bool ptlock_init(struct ptdesc *ptdesc){ /* * prep_new_page() initialize page->private (and therefore page->ptl) * with 0. Make sure nobody took it in use in between. * * It can happen if arch try to use slab for page table allocation: * slab code uses page->slab_cache, which share storage with page->ptl. */ VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); if (!ptlock_alloc(ptdesc)) return false; spin_lock_init(ptlock_ptr(ptdesc)); return true;}
#else /* !defined(CONFIG_SPLIT_PTE_PTLOCKS) *//* * We use mm->page_table_lock to guard all pagetable pages of the mm. */static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd){ return &mm->page_table_lock;}static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte){ return &mm->page_table_lock;}static inline void ptlock_cache_init(void) {}static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; }static inline void ptlock_free(struct ptdesc *ptdesc) {}#endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */
static inline void __pagetable_ctor(struct ptdesc *ptdesc){ struct folio *folio = ptdesc_folio(ptdesc);
__folio_set_pgtable(folio); lruvec_stat_add_folio(folio, NR_PAGETABLE);}
static inline void pagetable_dtor(struct ptdesc *ptdesc){ struct folio *folio = ptdesc_folio(ptdesc);
ptlock_free(ptdesc); __folio_clear_pgtable(folio); lruvec_stat_sub_folio(folio, NR_PAGETABLE);}
static inline void pagetable_dtor_free(struct ptdesc *ptdesc){ pagetable_dtor(ptdesc); pagetable_free(ptdesc);}
static inline bool pagetable_pte_ctor(struct mm_struct *mm, struct ptdesc *ptdesc){ if (mm != &init_mm && !ptlock_init(ptdesc)) return false; __pagetable_ctor(ptdesc); return true;}
pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr){ return __pte_offset_map(pmd, addr, NULL);}
pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp);
pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp);pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp, spinlock_t **ptlp);
#define pte_unmap_unlock(pte, ptl) do { \ spin_unlock(ptl); \ pte_unmap(pte); \} while (0)
#define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd))
#define pte_alloc_map(mm, pmd, address) \ (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address))
#define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
#define pte_alloc_kernel(pmd, address) \ ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ NULL: pte_offset_kernel(pmd, address))
#if defined(CONFIG_SPLIT_PMD_PTLOCKS)
static inline struct page *pmd_pgtable_page(pmd_t *pmd){ unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); return virt_to_page((void *)((unsigned long) pmd & mask));}
static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd){ return page_ptdesc(pmd_pgtable_page(pmd));}
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd){ return ptlock_ptr(pmd_ptdesc(pmd));}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc){#ifdef CONFIG_TRANSPARENT_HUGEPAGE ptdesc->pmd_huge_pte = NULL;#endif return ptlock_init(ptdesc);}
#define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
#else
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd){ return &mm->page_table_lock;}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
#endif
static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd){ spinlock_t *ptl = pmd_lockptr(mm, pmd); spin_lock(ptl); return ptl;}
static inline bool pagetable_pmd_ctor(struct mm_struct *mm, struct ptdesc *ptdesc){ if (mm != &init_mm && !pmd_ptlock_init(ptdesc)) return false; ptdesc_pmd_pts_init(ptdesc); __pagetable_ctor(ptdesc); return true;}
/* * No scalability reason to split PUD locks yet, but follow the same pattern * as the PMD locks to make it easier if we decide to. The VM should not be * considered ready to switch to split PUD locks yet; there may be places * which need to be converted from page_table_lock. */static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud){ return &mm->page_table_lock;}
static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud){ spinlock_t *ptl = pud_lockptr(mm, pud);
spin_lock(ptl); return ptl;}
static inline void pagetable_pud_ctor(struct ptdesc *ptdesc){ __pagetable_ctor(ptdesc);}
static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc){ __pagetable_ctor(ptdesc);}
static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc){ __pagetable_ctor(ptdesc);}
extern void __init pagecache_init(void);extern void free_initmem(void);
/* * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) * into the buddy system. The freed pages will be poisoned with pattern * "poison" if it's within range [0, UCHAR_MAX]. * Return pages freed into the buddy system. */extern unsigned long free_reserved_area(void *start, void *end, int poison, const char *s);
extern void adjust_managed_page_count(struct page *page, long count);
/* Free the reserved page into the buddy system, so it gets managed. */void free_reserved_page(struct page *page);
static inline void mark_page_reserved(struct page *page){ SetPageReserved(page); adjust_managed_page_count(page, -1);}
static inline void free_reserved_ptdesc(struct ptdesc *pt){ free_reserved_page(ptdesc_page(pt));}
/* * Default method to free all the __init memory into the buddy system. * The freed pages will be poisoned with pattern "poison" if it's within * range [0, UCHAR_MAX]. * Return pages freed into the buddy system. */static inline unsigned long free_initmem_default(int poison){ extern char __init_begin[], __init_end[];
return free_reserved_area(&__init_begin, &__init_end, poison, "unused kernel image (initmem)");}
static inline unsigned long get_num_physpages(void){ int nid; unsigned long phys_pages = 0;
for_each_online_node(nid) phys_pages += node_present_pages(nid);
return phys_pages;}
/* * FIXME: Using memblock node mappings, an architecture may initialise its * zones, allocate the backing mem_map and account for memory holes in an * architecture independent manner. * * An architecture is expected to register range of page frames backed by * physical memory with memblock_add[_node]() before calling * free_area_init() passing in the PFN each zone ends at. At a basic * usage, an architecture is expected to do something like * * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, * max_highmem_pfn}; * for_each_valid_physical_page_range() * memblock_add_node(base, size, nid, MEMBLOCK_NONE) * free_area_init(max_zone_pfns); */void arch_zone_limits_init(unsigned long *max_zone_pfn);unsigned long node_map_pfn_alignment(void);extern unsigned long absent_pages_in_range(unsigned long start_pfn, unsigned long end_pfn);extern void get_pfn_range_for_nid(unsigned int nid, unsigned long *start_pfn, unsigned long *end_pfn);
#ifndef CONFIG_NUMAstatic inline int early_pfn_to_nid(unsigned long pfn){ return 0;}#else/* please see mm/page_alloc.c */extern int __meminit early_pfn_to_nid(unsigned long pfn);#endif
extern void mem_init(void);extern void __init mmap_init(void);
extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);static inline void show_mem(void){ __show_mem(0, NULL, MAX_NR_ZONES - 1);}extern long si_mem_available(void);extern void si_meminfo(struct sysinfo * val);extern void si_meminfo_node(struct sysinfo *val, int nid);
extern __printf(3, 4)void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...);
extern void setup_per_cpu_pageset(void);
/* nommu.c */extern atomic_long_t mmap_pages_allocated;extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
/* interval_tree.c */void vma_interval_tree_insert(struct vm_area_struct *node, struct rb_root_cached *root);void vma_interval_tree_insert_after(struct vm_area_struct *node, struct vm_area_struct *prev, struct rb_root_cached *root);void vma_interval_tree_remove(struct vm_area_struct *node, struct rb_root_cached *root);struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, unsigned long start, unsigned long last);struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, unsigned long start, unsigned long last);struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, unsigned long start, unsigned long last);
#define vma_interval_tree_foreach(vma, root, start, last) \ for (vma = vma_interval_tree_iter_first(root, start, last); \ vma; vma = vma_interval_tree_iter_next(vma, start, last))
void anon_vma_interval_tree_insert(struct anon_vma_chain *node, struct rb_root_cached *root);void anon_vma_interval_tree_remove(struct anon_vma_chain *node, struct rb_root_cached *root);struct anon_vma_chain *anon_vma_interval_tree_iter_first(struct rb_root_cached *root, unsigned long start, unsigned long last);struct anon_vma_chain *anon_vma_interval_tree_iter_next( struct anon_vma_chain *node, unsigned long start, unsigned long last);#ifdef CONFIG_DEBUG_VM_RBvoid anon_vma_interval_tree_verify(struct anon_vma_chain *node);#endif
#define anon_vma_interval_tree_foreach(avc, root, start, last) \ for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
/* mmap.c */extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin);extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);extern void exit_mmap(struct mm_struct *);bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, bool write);
static inline int check_data_rlimit(unsigned long rlim, unsigned long new, unsigned long start, unsigned long end_data, unsigned long start_data){ if (rlim < RLIM_INFINITY) { if (((new - start) + (end_data - start_data)) > rlim) return -ENOSPC; }
return 0;}
extern int mm_take_all_locks(struct mm_struct *mm);extern void mm_drop_all_locks(struct mm_struct *mm);
extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);extern struct file *get_mm_exe_file(struct mm_struct *mm);extern struct file *get_task_exe_file(struct task_struct *task);
extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
extern bool vma_is_special_mapping(const struct vm_area_struct *vma, const struct vm_special_mapping *sm);struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, unsigned long addr, unsigned long len, vm_flags_t vm_flags, const struct vm_special_mapping *spec);
unsigned long randomize_stack_top(unsigned long stack_top);unsigned long randomize_page(unsigned long start, unsigned long range);
unsigned long__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags);
static inline unsigned longget_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags){ return __get_unmapped_area(file, addr, len, pgoff, flags, 0);}
extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf);extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock);int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, struct mm_struct *mm, unsigned long start, unsigned long end, struct list_head *uf, bool unlock);extern int do_munmap(struct mm_struct *, unsigned long, size_t, struct list_head *uf);extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);
#ifdef CONFIG_MMUextern int __mm_populate(unsigned long addr, unsigned long len, int ignore_errors);static inline void mm_populate(unsigned long addr, unsigned long len){ /* Ignore errors */ (void) __mm_populate(addr, len, 1);}#elsestatic inline void mm_populate(unsigned long addr, unsigned long len) {}#endif
/* This takes the mm semaphore itself */int __must_check vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec);int vm_munmap(unsigned long start, size_t len);unsigned long __must_check vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset);unsigned long __must_check vm_mmap_shadow_stack(unsigned long addr, unsigned long len, unsigned long flags);
struct vm_unmapped_area_info {#define VM_UNMAPPED_AREA_TOPDOWN 1 unsigned long flags; unsigned long length; unsigned long low_limit; unsigned long high_limit; unsigned long align_mask; unsigned long align_offset; unsigned long start_gap;};
extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info);
/* truncate.c */void truncate_inode_pages(struct address_space *mapping, loff_t lstart);void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart, uoff_t lend);void truncate_inode_pages_final(struct address_space *mapping);
/* generic vm_area_ops exported for stackable file systems */extern vm_fault_t filemap_fault(struct vm_fault *vmf);extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff);extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
extern unsigned long stack_guard_gap;/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */int expand_stack_locked(struct vm_area_struct *vma, unsigned long address);struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr);
/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, struct vm_area_struct **pprev);
/* * Look up the first VMA which intersects the interval [start_addr, end_addr) * NULL if none. Assume start_addr < end_addr. */struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, unsigned long start_addr, unsigned long end_addr);
/** * vma_lookup() - Find a VMA at a specific address * @mm: The process address space. * @addr: The user address. * * Return: The vm_area_struct at the given address, %NULL otherwise. */static inlinestruct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr){ return mtree_load(&mm->mm_mt, addr);}
static inline unsigned long stack_guard_start_gap(const struct vm_area_struct *vma){ if (vma->vm_flags & VM_GROWSDOWN) return stack_guard_gap;
/* See reasoning around the VM_SHADOW_STACK definition */ if (vma->vm_flags & VM_SHADOW_STACK) return PAGE_SIZE;
return 0;}
static inline unsigned long vm_start_gap(const struct vm_area_struct *vma){ unsigned long gap = stack_guard_start_gap(vma); unsigned long vm_start = vma->vm_start;
vm_start -= gap; if (vm_start > vma->vm_start) vm_start = 0; return vm_start;}
static inline unsigned long vm_end_gap(const struct vm_area_struct *vma){ unsigned long vm_end = vma->vm_end;
if (vma->vm_flags & VM_GROWSUP) { vm_end += stack_guard_gap; if (vm_end < vma->vm_end) vm_end = -PAGE_SIZE; } return vm_end;}
static inline unsigned long vma_pages(const struct vm_area_struct *vma){ return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;}
static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma){ return vma->vm_pgoff + vma_pages(vma) - 1;}
static inline unsigned long vma_desc_size(const struct vm_area_desc *desc){ return desc->end - desc->start;}
static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc){ return vma_desc_size(desc) >> PAGE_SHIFT;}
/** * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN * remap is required. * @desc: The VMA descriptor for the VMA requiring remap. * @start: The virtual address to start the remap from, must be within the VMA. * @start_pfn: The first PFN in the range to remap. * @size: The size of the range to remap, in bytes, at most spanning to the end * of the VMA. */static inline void mmap_action_remap(struct vm_area_desc *desc, unsigned long start, unsigned long start_pfn, unsigned long size){ struct mmap_action *action = &desc->action;
/* [start, start + size) must be within the VMA. */ WARN_ON_ONCE(start < desc->start || start >= desc->end); WARN_ON_ONCE(start + size > desc->end);
action->type = MMAP_REMAP_PFN; action->remap.start = start; action->remap.start_pfn = start_pfn; action->remap.size = size; action->remap.pgprot = desc->page_prot;}
/** * mmap_action_remap_full - helper for mmap_prepare hook to specify that the * entirety of a VMA should be PFN remapped. * @desc: The VMA descriptor for the VMA requiring remap. * @start_pfn: The first PFN in the range to remap. */static inline void mmap_action_remap_full(struct vm_area_desc *desc, unsigned long start_pfn){ mmap_action_remap(desc, desc->start, start_pfn, vma_desc_size(desc));}
/** * mmap_action_ioremap - helper for mmap_prepare hook to specify that a pure PFN * I/O remap is required. * @desc: The VMA descriptor for the VMA requiring remap. * @start: The virtual address to start the remap from, must be within the VMA. * @start_pfn: The first PFN in the range to remap. * @size: The size of the range to remap, in bytes, at most spanning to the end * of the VMA. */static inline void mmap_action_ioremap(struct vm_area_desc *desc, unsigned long start, unsigned long start_pfn, unsigned long size){ mmap_action_remap(desc, start, start_pfn, size); desc->action.type = MMAP_IO_REMAP_PFN;}
/** * mmap_action_ioremap_full - helper for mmap_prepare hook to specify that the * entirety of a VMA should be PFN I/O remapped. * @desc: The VMA descriptor for the VMA requiring remap. * @start_pfn: The first PFN in the range to remap. */static inline void mmap_action_ioremap_full(struct vm_area_desc *desc, unsigned long start_pfn){ mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc));}
/** * mmap_action_simple_ioremap - helper for mmap_prepare hook to specify that the * physical range in [start_phys_addr, start_phys_addr + size) should be I/O * remapped. * @desc: The VMA descriptor for the VMA requiring remap. * @start_phys_addr: Start of the physical memory to be mapped. * @size: Size of the area to map. * * NOTE: Some drivers might want to tweak desc->page_prot for purposes of * write-combine or similar. */static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc, phys_addr_t start_phys_addr, unsigned long size){ struct mmap_action *action = &desc->action;
action->simple_ioremap.start_phys_addr = start_phys_addr; action->simple_ioremap.size = size; action->type = MMAP_SIMPLE_IO_REMAP;}
/** * mmap_action_map_kernel_pages - helper for mmap_prepare hook to specify that * @num kernel pages contained in the @pages array should be mapped to userland * starting at virtual address @start. * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. * @start: The virtual address from which to map them. * @pages: An array of struct page pointers describing the memory to map. * @nr_pages: The number of entries in the @pages aray. */static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, unsigned long start, struct page **pages, unsigned long nr_pages){ struct mmap_action *action = &desc->action;
action->type = MMAP_MAP_KERNEL_PAGES; action->map_kernel.start = start; action->map_kernel.pages = pages; action->map_kernel.nr_pages = nr_pages; action->map_kernel.pgoff = desc->pgoff;}
/** * mmap_action_map_kernel_pages_full - helper for mmap_prepare hook to specify that * kernel pages contained in the @pages array should be mapped to userland * from @desc->start to @desc->end. * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. * @pages: An array of struct page pointers describing the memory to map. * * The caller must ensure that @pages contains sufficient entries to cover the * entire range described by @desc. */static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, struct page **pages){ mmap_action_map_kernel_pages(desc, desc->start, pages, vma_desc_pages(desc));}
int mmap_action_prepare(struct vm_area_desc *desc);int mmap_action_complete(struct vm_area_struct *vma, struct mmap_action *action, bool is_compat);
/* Look up the first VMA which exactly match the interval vm_start ... vm_end */static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, unsigned long vm_start, unsigned long vm_end){ struct vm_area_struct *vma = vma_lookup(mm, vm_start);
if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) vma = NULL;
return vma;}
/** * range_is_subset - Is the specified inner range a subset of the outer range? * @outer_start: The start of the outer range. * @outer_end: The exclusive end of the outer range. * @inner_start: The start of the inner range. * @inner_end: The exclusive end of the inner range. * * Returns: %true if [inner_start, inner_end) is a subset of [outer_start, * outer_end), otherwise %false. */static inline bool range_is_subset(unsigned long outer_start, unsigned long outer_end, unsigned long inner_start, unsigned long inner_end){ return outer_start <= inner_start && inner_end <= outer_end;}
/** * range_in_vma - is the specified [@start, @end) range a subset of the VMA? * @vma: The VMA against which we want to check [@start, @end). * @start: The start of the range we wish to check. * @end: The exclusive end of the range we wish to check. * * Returns: %true if [@start, @end) is a subset of [@vma->vm_start, * @vma->vm_end), %false otherwise. */static inline bool range_in_vma(const struct vm_area_struct *vma, unsigned long start, unsigned long end){ if (!vma) return false;
return range_is_subset(vma->vm_start, vma->vm_end, start, end);}
/** * range_in_vma_desc - is the specified [@start, @end) range a subset of the VMA * described by @desc, a VMA descriptor? * @desc: The VMA descriptor against which we want to check [@start, @end). * @start: The start of the range we wish to check. * @end: The exclusive end of the range we wish to check. * * Returns: %true if [@start, @end) is a subset of [@desc->start, @desc->end), * %false otherwise. */static inline bool range_in_vma_desc(const struct vm_area_desc *desc, unsigned long start, unsigned long end){ if (!desc) return false;
return range_is_subset(desc->start, desc->end, start, end);}
#ifdef CONFIG_MMUpgprot_t vm_get_page_prot(vm_flags_t vm_flags);
static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags){ const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags);
return vm_get_page_prot(vm_flags);}
void vma_set_page_prot(struct vm_area_struct *vma);#elsestatic inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags){ return __pgprot(0);}static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags){ return __pgprot(0);}static inline void vma_set_page_prot(struct vm_area_struct *vma){ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);}#endif
void vma_set_file(struct vm_area_struct *vma, struct file *file);
#ifdef CONFIG_NUMA_BALANCINGunsigned long change_prot_numa(struct vm_area_struct *vma, unsigned long start, unsigned long end);#endif
struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr);int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot);
int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num);int map_kernel_pages_prepare(struct vm_area_desc *desc);int map_kernel_pages_complete(struct vm_area_struct *vma, struct mmap_action *action);int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num);int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num);vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write);vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn);vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot);vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn);vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn);int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page){ int err = vm_insert_page(vma, addr, page);
if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;}
#ifndef io_remap_pfn_range_pfnstatic inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size){ return pfn;}#endif
static inline int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long orig_pfn, unsigned long size, pgprot_t orig_prot){ const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); const pgprot_t prot = pgprot_decrypted(orig_prot);
return remap_pfn_range(vma, addr, pfn, size, prot);}
static inline vm_fault_t vmf_error(int err){ if (err == -ENOMEM) return VM_FAULT_OOM; else if (err == -EHWPOISON) return VM_FAULT_HWPOISON; return VM_FAULT_SIGBUS;}
/* * Convert errno to return value for ->page_mkwrite() calls. * * This should eventually be merged with vmf_error() above, but will need a * careful audit of all vmf_error() callers. */static inline vm_fault_t vmf_fs_error(int err){ if (err == 0) return VM_FAULT_LOCKED; if (err == -EFAULT || err == -EAGAIN) return VM_FAULT_NOPAGE; if (err == -ENOMEM) return VM_FAULT_OOM; /* -ENOSPC, -EDQUOT, -EIO ... */ return VM_FAULT_SIGBUS;}
static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags){ if (vm_fault & VM_FAULT_OOM) return -ENOMEM; if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) return -EFAULT; return 0;}
/* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether * a (NUMA hinting) fault is required. */static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags){ /* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. */ if (!(flags & FOLL_HONOR_NUMA_FAULT)) return true;
/* * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. * * Requiring a fault here even for inaccessible VMAs would mean that * FOLL_FORCE cannot make any progress, because handle_mm_fault() * refuses to process NUMA hinting faults in inaccessible VMAs. */ return !vma_is_accessible(vma);}
typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data);extern int apply_to_existing_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data);
#ifdef CONFIG_PAGE_POISONINGextern void __kernel_poison_pages(struct page *page, int numpages);extern void __kernel_unpoison_pages(struct page *page, int numpages);extern bool _page_poisoning_enabled_early;DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled);static inline bool page_poisoning_enabled(void){ return _page_poisoning_enabled_early;}/* * For use in fast paths after init_mem_debugging() has run, or when a * false negative result is not harmful when called too early. */static inline bool page_poisoning_enabled_static(void){ return static_branch_unlikely(&_page_poisoning_enabled);}static inline void kernel_poison_pages(struct page *page, int numpages){ if (page_poisoning_enabled_static()) __kernel_poison_pages(page, numpages);}static inline void kernel_unpoison_pages(struct page *page, int numpages){ if (page_poisoning_enabled_static()) __kernel_unpoison_pages(page, numpages);}#elsestatic inline bool page_poisoning_enabled(void) { return false; }static inline bool page_poisoning_enabled_static(void) { return false; }static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }static inline void kernel_poison_pages(struct page *page, int numpages) { }static inline void kernel_unpoison_pages(struct page *page, int numpages) { }#endif
DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);static inline bool want_init_on_alloc(gfp_t flags){ if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, &init_on_alloc)) return true; return flags & __GFP_ZERO;}
DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);static inline bool want_init_on_free(void){ return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, &init_on_free);}
extern bool _debug_pagealloc_enabled_early;DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
static inline bool debug_pagealloc_enabled(void){ return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && _debug_pagealloc_enabled_early;}
/* * For use in fast paths after mem_debugging_and_hardening_init() has run, * or when a false negative result is not harmful when called too early. */static inline bool debug_pagealloc_enabled_static(void){ if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) return false;
return static_branch_unlikely(&_debug_pagealloc_enabled);}
/* * To support DEBUG_PAGEALLOC architecture must ensure that * __kernel_map_pages() never fails */extern void __kernel_map_pages(struct page *page, int numpages, int enable);#ifdef CONFIG_DEBUG_PAGEALLOCstatic inline void debug_pagealloc_map_pages(struct page *page, int numpages){ iommu_debug_check_unmapped(page, numpages);
if (debug_pagealloc_enabled_static()) __kernel_map_pages(page, numpages, 1);}
static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages){ iommu_debug_check_unmapped(page, numpages);
if (debug_pagealloc_enabled_static()) __kernel_map_pages(page, numpages, 0);}
extern unsigned int _debug_guardpage_minorder;DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
static inline unsigned int debug_guardpage_minorder(void){ return _debug_guardpage_minorder;}
static inline bool debug_guardpage_enabled(void){ return static_branch_unlikely(&_debug_guardpage_enabled);}
static inline bool page_is_guard(const struct page *page){ if (!debug_guardpage_enabled()) return false;
return PageGuard(page);}
bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order);static inline bool set_page_guard(struct zone *zone, struct page *page, unsigned int order){ if (!debug_guardpage_enabled()) return false; return __set_page_guard(zone, page, order);}
void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order);static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order){ if (!debug_guardpage_enabled()) return; __clear_page_guard(zone, page, order);}
#else /* CONFIG_DEBUG_PAGEALLOC */static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}static inline unsigned int debug_guardpage_minorder(void) { return 0; }static inline bool debug_guardpage_enabled(void) { return false; }static inline bool page_is_guard(const struct page *page) { return false; }static inline bool set_page_guard(struct zone *zone, struct page *page, unsigned int order) { return false; }static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order) {}#endif /* CONFIG_DEBUG_PAGEALLOC */
#ifndef clear_pages/** * clear_pages() - clear a page range for kernel-internal use. * @addr: start address * @npages: number of pages * * Use clear_user_pages() instead when clearing a page range to be * mapped to user space. * * Does absolutely no exception handling. * * Note that even though the clearing operation is preemptible, clear_pages() * does not (and on architectures where it reduces to a few long-running * instructions, might not be able to) call cond_resched() to check if * rescheduling is required. * * When running under preemptible models this is not a problem. Under * cooperatively scheduled models, however, the caller is expected to * limit @npages to no more than PROCESS_PAGES_NON_PREEMPT_BATCH. */static inline void clear_pages(void *addr, unsigned int npages){ do { clear_page(addr); addr += PAGE_SIZE; } while (--npages);}#endif
#ifndef PROCESS_PAGES_NON_PREEMPT_BATCH#ifdef clear_pages/* * The architecture defines clear_pages(), and we assume that it is * generally "fast". So choose a batch size large enough to allow the processor * headroom for optimizing the operation and yet small enough that we see * reasonable preemption latency for when this optimization is not possible * (ex. slow microarchitectures, memory bandwidth saturation.) * * With a value of 32MB and assuming a memory bandwidth of ~10GBps, this should * result in worst case preemption latency of around 3ms when clearing pages. * * (See comment above clear_pages() for why preemption latency is a concern * here.) */#define PROCESS_PAGES_NON_PREEMPT_BATCH (SZ_32M >> PAGE_SHIFT)#else /* !clear_pages *//* * The architecture does not provide a clear_pages() implementation. Assume * that clear_page() -- which clear_pages() will fallback to -- is relatively * slow and choose a small value for PROCESS_PAGES_NON_PREEMPT_BATCH. */#define PROCESS_PAGES_NON_PREEMPT_BATCH 1#endif#endif
#ifdef __HAVE_ARCH_GATE_AREAextern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);extern int in_gate_area_no_mm(unsigned long addr);extern int in_gate_area(struct mm_struct *mm, unsigned long addr);#elsestatic inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm){ return NULL;}static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }static inline int in_gate_area(struct mm_struct *mm, unsigned long addr){ return 0;}#endif /* __HAVE_ARCH_GATE_AREA */
bool process_shares_mm(const struct task_struct *p, const struct mm_struct *mm);
void drop_slab(void);
#ifndef CONFIG_MMU#define randomize_va_space 0#elseextern int randomize_va_space;#endif
const char * arch_vma_name(struct vm_area_struct *vma);#ifdef CONFIG_MMUvoid print_vma_addr(char *prefix, unsigned long rip);#elsestatic inline void print_vma_addr(char *prefix, unsigned long rip){}#endif
unsigned long section_map_size(void);struct page * __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap);void *vmemmap_alloc_block(unsigned long size, int node);struct vmem_altmap;void *vmemmap_alloc_block_buf(unsigned long size, int node, struct vmem_altmap *altmap);void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, unsigned long addr, unsigned long next);int vmemmap_check_pmd(pmd_t *pmd, int node, unsigned long addr, unsigned long next);int vmemmap_populate_basepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap);int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap);int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap);int vmemmap_populate_hvo(unsigned long start, unsigned long end, unsigned int order, struct zone *zone, unsigned long headsize);void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize);void vmemmap_populate_print_last(void);#ifdef CONFIG_MEMORY_HOTPLUGvoid vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap);#endif
#ifdef CONFIG_SPARSEMEM_VMEMMAPstatic inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap){ /* number of pfns from base where pfn_to_page() is valid */ if (altmap) return altmap->reserve + altmap->free; return 0;}
static inline void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns){ altmap->alloc -= nr_pfns;}#elsestatic inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap){ return 0;}
static inline void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns){}#endif
#define VMEMMAP_RESERVE_NR 2#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAPstatic inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap){ unsigned long nr_pages; unsigned long nr_vmemmap_pages;
if (!pgmap || !is_power_of_2(sizeof(struct page))) return false;
nr_pages = pgmap_vmemmap_nr(pgmap); nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); /* * For vmemmap optimization with DAX we need minimum 2 vmemmap * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst */ return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR);}/* * If we don't have an architecture override, use the generic rule */#ifndef vmemmap_can_optimize#define vmemmap_can_optimize __vmemmap_can_optimize#endif
#elsestatic inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap){ return false;}#endif
enum mf_flags { MF_COUNT_INCREASED = 1 << 0, MF_ACTION_REQUIRED = 1 << 1, MF_MUST_KILL = 1 << 2, MF_SOFT_OFFLINE = 1 << 3, MF_UNPOISON = 1 << 4, MF_SW_SIMULATED = 1 << 5, MF_NO_RETRY = 1 << 6, MF_MEM_PRE_REMOVE = 1 << 7,};int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, unsigned long count, int mf_flags);extern int memory_failure(unsigned long pfn, int flags);extern int unpoison_memory(unsigned long pfn);extern atomic_long_t num_poisoned_pages __read_mostly;extern int soft_offline_page(unsigned long pfn, int flags);#ifdef CONFIG_MEMORY_FAILURE/* * Sysfs entries for memory failure handling statistics. */extern const struct attribute_group memory_failure_attr_group;extern void memory_failure_queue(unsigned long pfn, int flags);void num_poisoned_pages_inc(unsigned long pfn);void num_poisoned_pages_sub(unsigned long pfn, long i);#elsestatic inline void memory_failure_queue(unsigned long pfn, int flags){}
static inline void num_poisoned_pages_inc(unsigned long pfn){}
static inline void num_poisoned_pages_sub(unsigned long pfn, long i){}#endif
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)extern void memblk_nr_poison_inc(unsigned long pfn);extern void memblk_nr_poison_sub(unsigned long pfn, long i);#elsestatic inline void memblk_nr_poison_inc(unsigned long pfn){}
static inline void memblk_nr_poison_sub(unsigned long pfn, long i){}#endif
#ifndef arch_memory_failurestatic inline int arch_memory_failure(unsigned long pfn, int flags){ return -ENXIO;}#endif
#ifndef arch_is_platform_pagestatic inline bool arch_is_platform_page(u64 paddr){ return false;}#endif
/* * Error handlers for various types of pages. */enum mf_result { MF_IGNORED, /* Error: cannot be handled */ MF_FAILED, /* Error: handling failed */ MF_DELAYED, /* Will be handled later */ MF_RECOVERED, /* Successfully recovered */};
enum mf_action_page_type { MF_MSG_KERNEL, MF_MSG_KERNEL_HIGH_ORDER, MF_MSG_DIFFERENT_COMPOUND, MF_MSG_HUGE, MF_MSG_FREE_HUGE, MF_MSG_GET_HWPOISON, MF_MSG_UNMAP_FAILED, MF_MSG_DIRTY_SWAPCACHE, MF_MSG_CLEAN_SWAPCACHE, MF_MSG_DIRTY_MLOCKED_LRU, MF_MSG_CLEAN_MLOCKED_LRU, MF_MSG_DIRTY_UNEVICTABLE_LRU, MF_MSG_CLEAN_UNEVICTABLE_LRU, MF_MSG_DIRTY_LRU, MF_MSG_CLEAN_LRU, MF_MSG_TRUNCATED_LRU, MF_MSG_BUDDY, MF_MSG_DAX, MF_MSG_UNSPLIT_THP, MF_MSG_ALREADY_POISONED, MF_MSG_PFN_MAP, MF_MSG_UNKNOWN,};
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)void folio_zero_user(struct folio *folio, unsigned long addr_hint);int copy_user_large_folio(struct folio *dst, struct folio *src, unsigned long addr_hint, struct vm_area_struct *vma);long copy_folio_from_user(struct folio *dst_folio, const void __user *usr_src, bool allow_pagefault);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
#if MAX_NUMNODES > 1void __init setup_nr_node_ids(void);#elsestatic inline void setup_nr_node_ids(void) {}#endif
extern int memcmp_pages(struct page *page1, struct page *page2);
static inline int pages_identical(struct page *page1, struct page *page2){ return !memcmp_pages(page1, page2);}
#ifdef CONFIG_MAPPING_DIRTY_HELPERSunsigned long clean_record_shared_mapping_range(struct address_space *mapping, pgoff_t first_index, pgoff_t nr, pgoff_t bitmap_pgoff, unsigned long *bitmap, pgoff_t *start, pgoff_t *end);
unsigned long wp_shared_mapping_range(struct address_space *mapping, pgoff_t first_index, pgoff_t nr);#endif
#ifdef CONFIG_ANON_VMA_NAMEint set_anon_vma_name(unsigned long addr, unsigned long size, const char __user *uname);#elsestatic inlineint set_anon_vma_name(unsigned long addr, unsigned long size, const char __user *uname){ return -EINVAL;}#endif
#ifdef CONFIG_UNACCEPTED_MEMORY
bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size);void accept_memory(phys_addr_t start, unsigned long size);
#else
static inline bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size){ return false;}
static inline void accept_memory(phys_addr_t start, unsigned long size){}
#endif
static inline bool pfn_is_unaccepted_memory(unsigned long pfn){ return range_contains_unaccepted_memory(pfn << PAGE_SHIFT, PAGE_SIZE);}
void vma_pgtable_walk_begin(struct vm_area_struct *vma);void vma_pgtable_walk_end(struct vm_area_struct *vma);
int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size);int reserve_mem_release_by_name(const char *name);
#ifdef CONFIG_64BITint do_mseal(unsigned long start, size_t len_in, unsigned long flags);#elsestatic inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags){ /* noop on 32 bit */ return 0;}#endif
/* * user_alloc_needs_zeroing checks if a user folio from page allocator needs to * be zeroed or not. */static inline bool user_alloc_needs_zeroing(void){ /* * for user folios, arch with cache aliasing requires cache flush and * arc changes folio->flags to make icache coherent with dcache, so * always return false to make caller use * clear_user_page()/clear_user_highpage(). */ return cpu_dcache_is_aliasing() || cpu_icache_is_aliasing() || !static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, &init_on_alloc);}
int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status);int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status);int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
/* * DMA mapping IDs for page_pool * * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and * stashes it in the upper bits of page->pp_magic. We always want to be able to * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP * pages can have arbitrary kernel pointers stored in the same field as pp_magic * (since it overlaps with page->lru.next), so we must ensure that we cannot * mistake a valid kernel pointer with any of the values we write into this * field. * * On architectures that set POISON_POINTER_DELTA, this is already ensured, * since this value becomes part of PP_SIGNATURE; meaning we can just use the * space between the PP_SIGNATURE value (without POISON_POINTER_DELTA), and the * lowest bits of POISON_POINTER_DELTA. On arches where POISON_POINTER_DELTA is * 0, we use the lowest bit of PAGE_OFFSET as the boundary if that value is * known at compile-time. * * If the value of PAGE_OFFSET is not known at compile time, or if it is too * small to leave at least 8 bits available above PP_SIGNATURE, we define the * number of bits to be 0, which turns off the DMA index tracking altogether * (see page_pool_register_dma_index()). */#define PP_DMA_INDEX_SHIFT (1 + __fls(PP_SIGNATURE - POISON_POINTER_DELTA))#if POISON_POINTER_DELTA > 0/* PP_SIGNATURE includes POISON_POINTER_DELTA, so limit the size of the DMA * index to not overlap with that if set */#define PP_DMA_INDEX_BITS MIN(32, __ffs(POISON_POINTER_DELTA) - PP_DMA_INDEX_SHIFT)#else/* Use the lowest bit of PAGE_OFFSET if there's at least 8 bits available; see above */#define PP_DMA_INDEX_MIN_OFFSET (1 << (PP_DMA_INDEX_SHIFT + 8))#define PP_DMA_INDEX_BITS ((__builtin_constant_p(PAGE_OFFSET) && \ PAGE_OFFSET >= PP_DMA_INDEX_MIN_OFFSET && \ !(PAGE_OFFSET & (PP_DMA_INDEX_MIN_OFFSET - 1))) ? \ MIN(32, __ffs(PAGE_OFFSET) - PP_DMA_INDEX_SHIFT) : 0)
#endif
#define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \ PP_DMA_INDEX_SHIFT)
/* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for * the head page of compound page and bit 1 for pfmemalloc page, as well as the * bits used for the DMA index. page_is_pfmemalloc() is checked in * __page_pool_put_page() to avoid recycling the pfmemalloc page. */#define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL)
#ifdef CONFIG_PAGE_POOLstatic inline bool page_pool_page_is_pp(const struct page *page){ return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;}#elsestatic inline bool page_pool_page_is_pp(const struct page *page){ return false;}#endif
#define PAGE_SNAPSHOT_FAITHFUL (1 << 0)#define PAGE_SNAPSHOT_PG_BUDDY (1 << 1)#define PAGE_SNAPSHOT_PG_IDLE (1 << 2)
struct page_snapshot { struct folio folio_snapshot; struct page page_snapshot; unsigned long pfn; unsigned long idx; unsigned long flags;};
static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps){ return ps->flags & PAGE_SNAPSHOT_FAITHFUL;}
void snapshot_page(struct page_snapshot *ps, const struct page *page);
void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, struct vm_area_struct *vma, unsigned long addr, bool uffd_wp);
#endif /* _LINUX_MM_H */