Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 1993 Linus Torvalds * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 * Numa awareness, Christoph Lameter, SGI, June 2005 * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 */
#include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/highmem.h>#include <linux/sched/signal.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/interrupt.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/set_memory.h>#include <linux/debugobjects.h>#include <linux/kallsyms.h>#include <linux/list.h>#include <linux/notifier.h>#include <linux/rbtree.h>#include <linux/xarray.h>#include <linux/io.h>#include <linux/rcupdate.h>#include <linux/pfn.h>#include <linux/kmemleak.h>#include <linux/atomic.h>#include <linux/compiler.h>#include <linux/memcontrol.h>#include <linux/llist.h>#include <linux/uio.h>#include <linux/bitops.h>#include <linux/rbtree_augmented.h>#include <linux/overflow.h>#include <linux/pgtable.h>#include <linux/hugetlb.h>#include <linux/sched/mm.h>#include <asm/tlbflush.h>#include <asm/shmparam.h>#include <linux/page_owner.h>#include <linux/cleanup.h>
#define CREATE_TRACE_POINTS#include <trace/events/vmalloc.h>
#include "internal.h"#include "pgalloc-track.h"#include "vmalloc.h"
#ifdef CONFIG_HAVE_ARCH_HUGE_VMAPstatic unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1;
static int __init set_nohugeiomap(char *str){ ioremap_max_page_shift = PAGE_SHIFT; return 0;}early_param("nohugeiomap", set_nohugeiomap);#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */static const unsigned int ioremap_max_page_shift = PAGE_SHIFT;#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOCstatic bool __ro_after_init vmap_allow_huge = true;
static int __init set_nohugevmalloc(char *str){ vmap_allow_huge = false; return 0;}early_param("nohugevmalloc", set_nohugevmalloc);#else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */static const bool vmap_allow_huge = false;#endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */
bool is_vmalloc_addr(const void *x){ unsigned long addr = (unsigned long)kasan_reset_tag(x);
return addr >= VMALLOC_START && addr < VMALLOC_END;}EXPORT_SYMBOL(is_vmalloc_addr);
struct vfree_deferred { struct llist_head list; struct work_struct wq;};static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
/*** Page table manipulation functions ***/static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift, pgtbl_mod_mask *mask){ pte_t *pte; u64 pfn; struct page *page; unsigned long size = PAGE_SIZE;
if (WARN_ON_ONCE(!PAGE_ALIGNED(end - addr))) return -EINVAL;
pfn = phys_addr >> PAGE_SHIFT; pte = pte_alloc_kernel_track(pmd, addr, mask); if (!pte) return -ENOMEM;
lazy_mmu_mode_enable();
do { if (unlikely(!pte_none(ptep_get(pte)))) { if (pfn_valid(pfn)) { page = pfn_to_page(pfn); dump_page(page, "remapping already mapped page"); } BUG(); }
#ifdef CONFIG_HUGETLB_PAGE size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); if (size != PAGE_SIZE) { pte_t entry = pfn_pte(pfn, prot);
entry = arch_make_huge_pte(entry, ilog2(size), 0); set_huge_pte_at(&init_mm, addr, pte, entry, size); pfn += PFN_DOWN(size); continue; }#endif set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); pfn++; } while (pte += PFN_DOWN(size), addr += size, addr != end);
lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED; return 0;}
static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift){ if (max_page_shift < PMD_SHIFT) return 0;
if (!arch_vmap_pmd_supported(prot)) return 0;
if ((end - addr) != PMD_SIZE) return 0;
if (!IS_ALIGNED(addr, PMD_SIZE)) return 0;
if (!IS_ALIGNED(phys_addr, PMD_SIZE)) return 0;
if (!pmd_present(*pmd)) return pmd_set_huge(pmd, phys_addr, prot);
/* * Acquire the mmap read lock to exclude ptdump, which walks * kernel page tables it does not own under the mmap write lock. * * Concurrent read lock holders are safe: each exclusively owns * the range it operates on and cannot reach this page table. */ scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { if (!pmd_free_pte_page(pmd, addr)) return 0; return pmd_set_huge(pmd, phys_addr, prot); }}
static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift, pgtbl_mod_mask *mask){ pmd_t *pmd; unsigned long next; int err = 0;
pmd = pmd_alloc_track(&init_mm, pud, addr, mask); if (!pmd) return -ENOMEM; do { next = pmd_addr_end(addr, end);
if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, max_page_shift)) { *mask |= PGTBL_PMD_MODIFIED; continue; }
err = vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask); if (err) break; } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); return err;}
static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift){ if (max_page_shift < PUD_SHIFT) return 0;
if (!arch_vmap_pud_supported(prot)) return 0;
if ((end - addr) != PUD_SIZE) return 0;
if (!IS_ALIGNED(addr, PUD_SIZE)) return 0;
if (!IS_ALIGNED(phys_addr, PUD_SIZE)) return 0;
if (!pud_present(*pud)) return pud_set_huge(pud, phys_addr, prot);
/* See comment in vmap_try_huge_pmd(). */ scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { if (!pud_free_pmd_page(pud, addr)) return 0; return pud_set_huge(pud, phys_addr, prot); }}
static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift, pgtbl_mod_mask *mask){ pud_t *pud; unsigned long next; int err = 0;
pud = pud_alloc_track(&init_mm, p4d, addr, mask); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end);
if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, max_page_shift)) { *mask |= PGTBL_PUD_MODIFIED; continue; }
err = vmap_pmd_range(pud, addr, next, phys_addr, prot, max_page_shift, mask); if (err) break; } while (pud++, phys_addr += (next - addr), addr = next, addr != end); return err;}
static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift){ if (max_page_shift < P4D_SHIFT) return 0;
if (!arch_vmap_p4d_supported(prot)) return 0;
if ((end - addr) != P4D_SIZE) return 0;
if (!IS_ALIGNED(addr, P4D_SIZE)) return 0;
if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0;
if (!p4d_present(*p4d)) return p4d_set_huge(p4d, phys_addr, prot);
/* See comment in vmap_try_huge_pmd(). */ scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { if (!p4d_free_pud_page(p4d, addr)) return 0; return p4d_set_huge(p4d, phys_addr, prot); }}
static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift, pgtbl_mod_mask *mask){ p4d_t *p4d; unsigned long next; int err = 0;
p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end);
if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, max_page_shift)) { *mask |= PGTBL_P4D_MODIFIED; continue; }
err = vmap_pud_range(p4d, addr, next, phys_addr, prot, max_page_shift, mask); if (err) break; } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); return err;}
static int vmap_range_noflush(unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot, unsigned int max_page_shift){ pgd_t *pgd; unsigned long start; unsigned long next; int err; pgtbl_mod_mask mask = 0;
/* * Might allocate pagetables (for most archs a more precise annotation * would be might_alloc(GFP_PGTABLE_KERNEL)). Also might shootdown TLB * (requires IRQs enabled on x86). */ might_sleep(); BUG_ON(addr >= end);
start = addr; pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, max_page_shift, &mask); if (err) break; } while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK) arch_sync_kernel_mappings(start, end);
return err;}
int vmap_page_range(unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot){ int err;
err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), ioremap_max_page_shift); flush_cache_vmap(addr, end); if (!err) err = kmsan_ioremap_page_range(addr, end, phys_addr, prot, ioremap_max_page_shift); return err;}
int ioremap_page_range(unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot){ struct vm_struct *area;
area = find_vm_area((void *)addr); if (!area || !(area->flags & VM_IOREMAP)) { WARN_ONCE(1, "vm_area at addr %lx is not marked as VM_IOREMAP\n", addr); return -EINVAL; } if (addr != (unsigned long)area->addr || (void *)end != area->addr + get_vm_area_size(area)) { WARN_ONCE(1, "ioremap request [%lx,%lx) doesn't match vm_area [%lx, %lx)\n", addr, end, (long)area->addr, (long)area->addr + get_vm_area_size(area)); return -ERANGE; } return vmap_page_range(addr, end, phys_addr, prot);}
static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pgtbl_mod_mask *mask){ pte_t *pte; pte_t ptent; unsigned long size = PAGE_SIZE;
pte = pte_offset_kernel(pmd, addr); lazy_mmu_mode_enable();
do {#ifdef CONFIG_HUGETLB_PAGE size = arch_vmap_pte_range_unmap_size(addr, pte); if (size != PAGE_SIZE) { if (WARN_ON(!IS_ALIGNED(addr, size))) { addr = ALIGN_DOWN(addr, size); pte = PTR_ALIGN_DOWN(pte, sizeof(*pte) * (size >> PAGE_SHIFT)); } ptent = huge_ptep_get_and_clear(&init_mm, addr, pte, size); if (WARN_ON(end - addr < size)) size = end - addr; } else#endif ptent = ptep_get_and_clear(&init_mm, addr, pte); WARN_ON(!pte_none(ptent) && !pte_present(ptent)); } while (pte += (size >> PAGE_SHIFT), addr += size, addr != end);
lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED;}
static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pgtbl_mod_mask *mask){ pmd_t *pmd; unsigned long next; int cleared;
pmd = pmd_offset(pud, addr); do { next = pmd_addr_end(addr, end);
cleared = pmd_clear_huge(pmd); if (cleared || pmd_bad(*pmd)) *mask |= PGTBL_PMD_MODIFIED;
if (cleared) { WARN_ON(next - addr < PMD_SIZE); continue; } if (pmd_none_or_clear_bad(pmd)) continue; vunmap_pte_range(pmd, addr, next, mask);
cond_resched(); } while (pmd++, addr = next, addr != end);}
static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgtbl_mod_mask *mask){ pud_t *pud; unsigned long next; int cleared;
pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end);
cleared = pud_clear_huge(pud); if (cleared || pud_bad(*pud)) *mask |= PGTBL_PUD_MODIFIED;
if (cleared) { WARN_ON(next - addr < PUD_SIZE); continue; } if (pud_none_or_clear_bad(pud)) continue; vunmap_pmd_range(pud, addr, next, mask); } while (pud++, addr = next, addr != end);}
static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, pgtbl_mod_mask *mask){ p4d_t *p4d; unsigned long next;
p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end);
p4d_clear_huge(p4d); if (p4d_bad(*p4d)) *mask |= PGTBL_P4D_MODIFIED;
if (p4d_none_or_clear_bad(p4d)) continue; vunmap_pud_range(p4d, addr, next, mask); } while (p4d++, addr = next, addr != end);}
/* * vunmap_range_noflush is similar to vunmap_range, but does not * flush caches or TLBs. * * The caller is responsible for calling flush_cache_vmap() before calling * this function, and flush_tlb_kernel_range after it has returned * successfully (and before the addresses are expected to cause a page fault * or be re-mapped for something else, if TLB flushes are being delayed or * coalesced). * * This is an internal function only. Do not use outside mm/. */void __vunmap_range_noflush(unsigned long start, unsigned long end){ unsigned long next; pgd_t *pgd; unsigned long addr = start; pgtbl_mod_mask mask = 0;
BUG_ON(addr >= end); pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); if (pgd_bad(*pgd)) mask |= PGTBL_PGD_MODIFIED; if (pgd_none_or_clear_bad(pgd)) continue; vunmap_p4d_range(pgd, addr, next, &mask); } while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK) arch_sync_kernel_mappings(start, end);}
void vunmap_range_noflush(unsigned long start, unsigned long end){ kmsan_vunmap_range_noflush(start, end); __vunmap_range_noflush(start, end);}
/** * vunmap_range - unmap kernel virtual addresses * @addr: start of the VM area to unmap * @end: end of the VM area to unmap (non-inclusive) * * Clears any present PTEs in the virtual address range, flushes TLBs and * caches. Any subsequent access to the address before it has been re-mapped * is a kernel bug. */void vunmap_range(unsigned long addr, unsigned long end){ flush_cache_vunmap(addr, end); vunmap_range_noflush(addr, end); flush_tlb_kernel_range(addr, end);}
static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, pgtbl_mod_mask *mask){ int err = 0; pte_t *pte;
/* * nr is a running index into the array which helps higher level * callers keep track of where we're up to. */
pte = pte_alloc_kernel_track(pmd, addr, mask); if (!pte) return -ENOMEM;
lazy_mmu_mode_enable();
do { struct page *page = pages[*nr];
if (WARN_ON(!pte_none(ptep_get(pte)))) { err = -EBUSY; break; } if (WARN_ON(!page)) { err = -ENOMEM; break; } if (WARN_ON(!pfn_valid(page_to_pfn(page)))) { err = -EINVAL; break; }
set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); (*nr)++; } while (pte++, addr += PAGE_SIZE, addr != end);
lazy_mmu_mode_disable(); *mask |= PGTBL_PTE_MODIFIED;
return err;}
static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, pgtbl_mod_mask *mask){ pmd_t *pmd; unsigned long next;
pmd = pmd_alloc_track(&init_mm, pud, addr, mask); if (!pmd) return -ENOMEM; do { next = pmd_addr_end(addr, end); if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) return -ENOMEM; } while (pmd++, addr = next, addr != end); return 0;}
static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, pgtbl_mod_mask *mask){ pud_t *pud; unsigned long next;
pud = pud_alloc_track(&init_mm, p4d, addr, mask); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) return -ENOMEM; } while (pud++, addr = next, addr != end); return 0;}
static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, pgtbl_mod_mask *mask){ p4d_t *p4d; unsigned long next;
p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end); if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) return -ENOMEM; } while (p4d++, addr = next, addr != end); return 0;}
static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages){ unsigned long start = addr; pgd_t *pgd; unsigned long next; int err = 0; int nr = 0; pgtbl_mod_mask mask = 0;
BUG_ON(addr >= end); pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); if (pgd_bad(*pgd)) mask |= PGTBL_PGD_MODIFIED; err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); if (err) break; } while (pgd++, addr = next, addr != end);
if (mask & ARCH_PAGE_TABLE_SYNC_MASK) arch_sync_kernel_mappings(start, end);
return err;}
/* * vmap_pages_range_noflush is similar to vmap_pages_range, but does not * flush caches. * * The caller is responsible for calling flush_cache_vmap() after this * function returns successfully and before the addresses are accessed. * * This is an internal function only. Do not use outside mm/. */int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift){ unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
if (WARN_ON_ONCE(page_shift < PAGE_SHIFT)) return -EINVAL;
if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || page_shift == PAGE_SHIFT) return vmap_small_pages_range_noflush(addr, end, prot, pages);
for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { int err;
err = vmap_range_noflush(addr, addr + (1UL << page_shift), page_to_phys(pages[i]), prot, page_shift); if (err) return err;
addr += 1UL << page_shift; }
return 0;}
int vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask){ int ret = kmsan_vmap_pages_range_noflush(addr, end, prot, pages, page_shift, gfp_mask);
if (ret) return ret; return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);}
static int __vmap_pages_range(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask){ int err;
err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift, gfp_mask); flush_cache_vmap(addr, end); return err;}
/** * vmap_pages_range - map pages to a kernel virtual address * @addr: start of the VM area to map * @end: end of the VM area to map (non-inclusive) * @prot: page protection flags to use * @pages: pages to map (always PAGE_SIZE pages) * @page_shift: maximum shift that the pages may be mapped with, @pages must * be aligned and contiguous up to at least this shift. * * RETURNS: * 0 on success, -errno on failure. */int vmap_pages_range(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift){ return __vmap_pages_range(addr, end, prot, pages, page_shift, GFP_KERNEL);}
static int check_sparse_vm_area(struct vm_struct *area, unsigned long start, unsigned long end){ might_sleep(); if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS)) return -EINVAL; if (WARN_ON_ONCE(area->flags & VM_NO_GUARD)) return -EINVAL; if (WARN_ON_ONCE(!(area->flags & VM_SPARSE))) return -EINVAL; if ((end - start) >> PAGE_SHIFT > totalram_pages()) return -E2BIG; if (start < (unsigned long)area->addr || (void *)end > area->addr + get_vm_area_size(area)) return -ERANGE; return 0;}
/** * vm_area_map_pages - map pages inside given sparse vm_area * @area: vm_area * @start: start address inside vm_area * @end: end address inside vm_area * @pages: pages to map (always PAGE_SIZE pages) */int vm_area_map_pages(struct vm_struct *area, unsigned long start, unsigned long end, struct page **pages){ int err;
err = check_sparse_vm_area(area, start, end); if (err) return err;
return vmap_pages_range(start, end, PAGE_KERNEL, pages, PAGE_SHIFT);}
/** * vm_area_unmap_pages - unmap pages inside given sparse vm_area * @area: vm_area * @start: start address inside vm_area * @end: end address inside vm_area */void vm_area_unmap_pages(struct vm_struct *area, unsigned long start, unsigned long end){ if (check_sparse_vm_area(area, start, end)) return;
vunmap_range(start, end);}
int is_vmalloc_or_module_addr(const void *x){ /* * ARM, x86-64 and sparc64 put modules in a special place, * and fall back on vmalloc() if that fails. Others * just put it in the vmalloc space. */#if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR) unsigned long addr = (unsigned long)kasan_reset_tag(x); if (addr >= MODULES_VADDR && addr < MODULES_END) return 1;#endif return is_vmalloc_addr(x);}EXPORT_SYMBOL_GPL(is_vmalloc_or_module_addr);
/* * Walk a vmap address to the struct page it maps. Huge vmap mappings will * return the tail page that corresponds to the base page address, which * matches small vmap mappings. Unlike vmalloc_to_page(), this also accepts * addresses outside the vmalloc and module ranges, such as KMSAN metadata. */__always_inline struct page *__vmalloc_to_page(const void *vmalloc_addr){ unsigned long addr = (unsigned long) vmalloc_addr; struct page *page = NULL; pgd_t *pgd = pgd_offset_k(addr); p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *ptep, pte;
if (pgd_none(*pgd)) return NULL; if (WARN_ON_ONCE(pgd_leaf(*pgd))) return NULL; /* XXX: no allowance for huge pgd */ if (WARN_ON_ONCE(pgd_bad(*pgd))) return NULL;
p4d = p4d_offset(pgd, addr); if (p4d_none(*p4d)) return NULL; if (p4d_leaf(*p4d)) return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); if (WARN_ON_ONCE(p4d_bad(*p4d))) return NULL;
pud = pud_offset(p4d, addr); if (pud_none(*pud)) return NULL; if (pud_leaf(*pud)) return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); if (WARN_ON_ONCE(pud_bad(*pud))) return NULL;
pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) return NULL; if (pmd_leaf(*pmd)) return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); if (WARN_ON_ONCE(pmd_bad(*pmd))) return NULL;
ptep = pte_offset_kernel(pmd, addr); pte = ptep_get(ptep); if (pte_present(pte)) page = pte_page(pte);
return page;}
struct page *vmalloc_to_page(const void *vmalloc_addr){ /* * XXX we might need to change this if we add VIRTUAL_BUG_ON for * architectures that do not vmalloc module space */ VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); return __vmalloc_to_page(vmalloc_addr);}EXPORT_SYMBOL(vmalloc_to_page);
/* * Map a vmalloc()-space virtual address to the physical page frame number. */unsigned long vmalloc_to_pfn(const void *vmalloc_addr){ return page_to_pfn(vmalloc_to_page(vmalloc_addr));}EXPORT_SYMBOL(vmalloc_to_pfn);
/*** Global kva allocator ***/
#define DEBUG_AUGMENT_PROPAGATE_CHECK 0#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
static DEFINE_SPINLOCK(free_vmap_area_lock);static bool vmap_initialized __read_mostly;
/* * This kmem_cache is used for vmap_area objects. Instead of * allocating from slab we reuse an object from this cache to * make things faster. Especially in "no edge" splitting of * free block. */static struct kmem_cache *vmap_area_cachep;
/* * This linked list is used in pair with free_vmap_area_root. * It gives O(1) access to prev/next to perform fast coalescing. */static LIST_HEAD(free_vmap_area_list);
/* * This augment red-black tree represents the free vmap space. * All vmap_area objects in this tree are sorted by va->va_start * address. It is used for allocation and merging when a vmap * object is released. * * Each vmap_area node contains a maximum available free block * of its sub-tree, right or left. Therefore it is possible to * find a lowest match of free area. */static struct rb_root free_vmap_area_root = RB_ROOT;
/* * Preload a CPU with one object for "no edge" split case. The * aim is to get rid of allocations from the atomic context, thus * to use more permissive allocation masks. */static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node);
/* * This structure defines a single, solid model where a list and * rb-tree are part of one entity protected by the lock. Nodes are * sorted in ascending order, thus for O(1) access to left/right * neighbors a list is used as well as for sequential traversal. */struct rb_list { struct rb_root root; struct list_head head; spinlock_t lock;};
/* * A fast size storage contains VAs up to 1M size. A pool consists * of linked between each other ready to go VAs of certain sizes. * An index in the pool-array corresponds to number of pages + 1. */#define MAX_VA_SIZE_PAGES 256
struct vmap_pool { struct list_head head; unsigned long len;};
/* * An effective vmap-node logic. Users make use of nodes instead * of a global heap. It allows to balance an access and mitigate * contention. */static struct vmap_node { /* Simple size segregated storage. */ struct vmap_pool pool[MAX_VA_SIZE_PAGES]; spinlock_t pool_lock; bool skip_populate;
/* Bookkeeping data of this node. */ struct rb_list busy; struct rb_list lazy;
/* * Ready-to-free areas. */ struct list_head purge_list; struct work_struct purge_work; unsigned long nr_purged;} single;
/* * Initial setup consists of one single node, i.e. a balancing * is fully disabled. Later on, after vmap is initialized these * parameters are updated based on a system capacity. */static struct vmap_node *vmap_nodes = &single;static __read_mostly unsigned int nr_vmap_nodes = 1;static __read_mostly unsigned int vmap_zone_size = 1;
/* A simple iterator over all vmap-nodes. */#define for_each_vmap_node(vn) \ for ((vn) = &vmap_nodes[0]; \ (vn) < &vmap_nodes[nr_vmap_nodes]; (vn)++)
static inline unsigned intaddr_to_node_id(unsigned long addr){ return (addr / vmap_zone_size) % nr_vmap_nodes;}
static inline struct vmap_node *addr_to_node(unsigned long addr){ return &vmap_nodes[addr_to_node_id(addr)];}
static inline struct vmap_node *id_to_node(unsigned int id){ return &vmap_nodes[id % nr_vmap_nodes];}
static inline unsigned intnode_to_id(struct vmap_node *node){ /* Pointer arithmetic. */ unsigned int id = node - vmap_nodes;
if (likely(id < nr_vmap_nodes)) return id;
WARN_ONCE(1, "An address 0x%p is out-of-bounds.\n", node); return 0;}
/* * We use the value 0 to represent "no node", that is why * an encoded value will be the node-id incremented by 1. * It is always greater then 0. A valid node_id which can * be encoded is [0:nr_vmap_nodes - 1]. If a passed node_id * is not valid 0 is returned. */static unsigned intencode_vn_id(unsigned int node_id){ /* Can store U8_MAX [0:254] nodes. */ if (node_id < nr_vmap_nodes) return (node_id + 1) << BITS_PER_BYTE;
/* Warn and no node encoded. */ WARN_ONCE(1, "Encode wrong node id (%u)\n", node_id); return 0;}
/* * Returns an encoded node-id, the valid range is within * [0:nr_vmap_nodes-1] values. Otherwise nr_vmap_nodes is * returned if extracted data is wrong. */static unsigned intdecode_vn_id(unsigned int val){ unsigned int node_id = (val >> BITS_PER_BYTE) - 1;
/* Can store U8_MAX [0:254] nodes. */ if (node_id < nr_vmap_nodes) return node_id;
/* If it was _not_ zero, warn. */ WARN_ONCE(node_id != UINT_MAX, "Decode wrong node id (%d)\n", node_id);
return nr_vmap_nodes;}
static boolis_vn_id_valid(unsigned int node_id){ if (node_id < nr_vmap_nodes) return true;
return false;}
static __always_inline unsigned longva_size(struct vmap_area *va){ return (va->va_end - va->va_start);}
static __always_inline unsigned longget_subtree_max_size(struct rb_node *node){ struct vmap_area *va;
va = rb_entry_safe(node, struct vmap_area, rb_node); return va ? va->subtree_max_size : 0;}
RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size)
static void reclaim_and_purge_vmap_areas(void);static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);static void drain_vmap_area_work(struct work_struct *work);static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work);
static __cacheline_aligned_in_smp atomic_long_t vmap_lazy_nr;
static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root){ struct rb_node *n = root->rb_node;
addr = (unsigned long)kasan_reset_tag((void *)addr);
while (n) { struct vmap_area *va;
va = rb_entry(n, struct vmap_area, rb_node); if (addr < va->va_start) n = n->rb_left; else if (addr >= va->va_end) n = n->rb_right; else return va; }
return NULL;}
/* Look up the first VA which satisfies addr < va_end, NULL if none. */static struct vmap_area *__find_vmap_area_exceed_addr(unsigned long addr, struct rb_root *root){ struct vmap_area *va = NULL; struct rb_node *n = root->rb_node;
addr = (unsigned long)kasan_reset_tag((void *)addr);
while (n) { struct vmap_area *tmp;
tmp = rb_entry(n, struct vmap_area, rb_node); if (tmp->va_end > addr) { va = tmp; if (tmp->va_start <= addr) break;
n = n->rb_left; } else n = n->rb_right; }
return va;}
/* * Returns a node where a first VA, that satisfies addr < va_end, resides. * If success, a node is locked. A user is responsible to unlock it when a * VA is no longer needed to be accessed. * * Returns NULL if nothing found. */static struct vmap_node *find_vmap_area_exceed_addr_lock(unsigned long addr, struct vmap_area **va){ unsigned long va_start_lowest; struct vmap_node *vn;
repeat: va_start_lowest = 0;
for_each_vmap_node(vn) { spin_lock(&vn->busy.lock); *va = __find_vmap_area_exceed_addr(addr, &vn->busy.root);
if (*va) if (!va_start_lowest || (*va)->va_start < va_start_lowest) va_start_lowest = (*va)->va_start; spin_unlock(&vn->busy.lock); }
/* * Check if found VA exists, it might have gone away. In this case we * repeat the search because a VA has been removed concurrently and we * need to proceed to the next one, which is a rare case. */ if (va_start_lowest) { vn = addr_to_node(va_start_lowest);
spin_lock(&vn->busy.lock); *va = __find_vmap_area(va_start_lowest, &vn->busy.root);
if (*va) return vn;
spin_unlock(&vn->busy.lock); goto repeat; }
return NULL;}
/* * This function returns back addresses of parent node * and its left or right link for further processing. * * Otherwise NULL is returned. In that case all further * steps regarding inserting of conflicting overlap range * have to be declined and actually considered as a bug. */static __always_inline struct rb_node **find_va_links(struct vmap_area *va, struct rb_root *root, struct rb_node *from, struct rb_node **parent){ struct vmap_area *tmp_va; struct rb_node **link;
if (root) { link = &root->rb_node; if (unlikely(!*link)) { *parent = NULL; return link; } } else { link = &from; }
/* * Go to the bottom of the tree. When we hit the last point * we end up with parent rb_node and correct direction, i name * it link, where the new va->rb_node will be attached to. */ do { tmp_va = rb_entry(*link, struct vmap_area, rb_node);
/* * During the traversal we also do some sanity check. * Trigger the BUG() if there are sides(left/right) * or full overlaps. */ if (va->va_end <= tmp_va->va_start) link = &(*link)->rb_left; else if (va->va_start >= tmp_va->va_end) link = &(*link)->rb_right; else { WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n", va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end);
return NULL; } } while (*link);
*parent = &tmp_va->rb_node; return link;}
static __always_inline struct list_head *get_va_next_sibling(struct rb_node *parent, struct rb_node **link){ struct list_head *list;
if (unlikely(!parent)) /* * The red-black tree where we try to find VA neighbors * before merging or inserting is empty, i.e. it means * there is no free vmap space. Normally it does not * happen but we handle this case anyway. */ return NULL;
list = &rb_entry(parent, struct vmap_area, rb_node)->list; return (&parent->rb_right == link ? list->next : list);}
static __always_inline void__link_va(struct vmap_area *va, struct rb_root *root, struct rb_node *parent, struct rb_node **link, struct list_head *head, bool augment){ /* * VA is still not in the list, but we can * identify its future previous list_head node. */ if (likely(parent)) { head = &rb_entry(parent, struct vmap_area, rb_node)->list; if (&parent->rb_right != link) head = head->prev; }
/* Insert to the rb-tree */ rb_link_node(&va->rb_node, parent, link); if (augment) { /* * Some explanation here. Just perform simple insertion * to the tree. We do not set va->subtree_max_size to * its current size before calling rb_insert_augmented(). * It is because we populate the tree from the bottom * to parent levels when the node _is_ in the tree. * * Therefore we set subtree_max_size to zero after insertion, * to let __augment_tree_propagate_from() puts everything to * the correct order later on. */ rb_insert_augmented(&va->rb_node, root, &free_vmap_area_rb_augment_cb); va->subtree_max_size = 0; } else { rb_insert_color(&va->rb_node, root); }
/* Address-sort this list */ list_add(&va->list, head);}
static __always_inline voidlink_va(struct vmap_area *va, struct rb_root *root, struct rb_node *parent, struct rb_node **link, struct list_head *head){ __link_va(va, root, parent, link, head, false);}
static __always_inline voidlink_va_augment(struct vmap_area *va, struct rb_root *root, struct rb_node *parent, struct rb_node **link, struct list_head *head){ __link_va(va, root, parent, link, head, true);}
static __always_inline void__unlink_va(struct vmap_area *va, struct rb_root *root, bool augment){ if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) return;
if (augment) rb_erase_augmented(&va->rb_node, root, &free_vmap_area_rb_augment_cb); else rb_erase(&va->rb_node, root);
list_del_init(&va->list); RB_CLEAR_NODE(&va->rb_node);}
static __always_inline voidunlink_va(struct vmap_area *va, struct rb_root *root){ __unlink_va(va, root, false);}
static __always_inline voidunlink_va_augment(struct vmap_area *va, struct rb_root *root){ __unlink_va(va, root, true);}
#if DEBUG_AUGMENT_PROPAGATE_CHECK/* * Gets called when remove the node and rotate. */static __always_inline unsigned longcompute_subtree_max_size(struct vmap_area *va){ return max3(va_size(va), get_subtree_max_size(va->rb_node.rb_left), get_subtree_max_size(va->rb_node.rb_right));}
static voidaugment_tree_propagate_check(void){ struct vmap_area *va; unsigned long computed_size;
list_for_each_entry(va, &free_vmap_area_list, list) { computed_size = compute_subtree_max_size(va); if (computed_size != va->subtree_max_size) pr_emerg("tree is corrupted: %lu, %lu\n", va_size(va), va->subtree_max_size); }}#endif
/* * This function populates subtree_max_size from bottom to upper * levels starting from VA point. The propagation must be done * when VA size is modified by changing its va_start/va_end. Or * in case of newly inserting of VA to the tree. * * It means that __augment_tree_propagate_from() must be called: * - After VA has been inserted to the tree(free path); * - After VA has been shrunk(allocation path); * - After VA has been increased(merging path). * * Please note that, it does not mean that upper parent nodes * and their subtree_max_size are recalculated all the time up * to the root node. * * 4--8 * /\ * / \ * / \ * 2--2 8--8 * * For example if we modify the node 4, shrinking it to 2, then * no any modification is required. If we shrink the node 2 to 1 * its subtree_max_size is updated only, and set to 1. If we shrink * the node 8 to 6, then its subtree_max_size is set to 6 and parent * node becomes 4--6. */static __always_inline voidaugment_tree_propagate_from(struct vmap_area *va){ /* * Populate the tree from bottom towards the root until * the calculated maximum available size of checked node * is equal to its current one. */ free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL);
#if DEBUG_AUGMENT_PROPAGATE_CHECK augment_tree_propagate_check();#endif}
static voidinsert_vmap_area(struct vmap_area *va, struct rb_root *root, struct list_head *head){ struct rb_node **link; struct rb_node *parent;
link = find_va_links(va, root, NULL, &parent); if (link) link_va(va, root, parent, link, head);}
static voidinsert_vmap_area_augment(struct vmap_area *va, struct rb_node *from, struct rb_root *root, struct list_head *head){ struct rb_node **link; struct rb_node *parent;
if (from) link = find_va_links(va, NULL, from, &parent); else link = find_va_links(va, root, NULL, &parent);
if (link) { link_va_augment(va, root, parent, link, head); augment_tree_propagate_from(va); }}
/* * Merge de-allocated chunk of VA memory with previous * and next free blocks. If coalesce is not done a new * free area is inserted. If VA has been merged, it is * freed. * * Please note, it can return NULL in case of overlap * ranges, followed by WARN() report. Despite it is a * buggy behaviour, a system can be alive and keep * ongoing. */static __always_inline struct vmap_area *__merge_or_add_vmap_area(struct vmap_area *va, struct rb_root *root, struct list_head *head, bool augment){ struct vmap_area *sibling; struct list_head *next; struct rb_node **link; struct rb_node *parent; bool merged = false;
/* * Find a place in the tree where VA potentially will be * inserted, unless it is merged with its sibling/siblings. */ link = find_va_links(va, root, NULL, &parent); if (!link) return NULL;
/* * Get next node of VA to check if merging can be done. */ next = get_va_next_sibling(parent, link); if (unlikely(next == NULL)) goto insert;
/* * start end * | | * |<------VA------>|<-----Next----->| * | | * start end */ if (next != head) { sibling = list_entry(next, struct vmap_area, list); if (sibling->va_start == va->va_end) { sibling->va_start = va->va_start;
/* Free vmap_area object. */ kmem_cache_free(vmap_area_cachep, va);
/* Point to the new merged area. */ va = sibling; merged = true; } }
/* * start end * | | * |<-----Prev----->|<------VA------>| * | | * start end */ if (next->prev != head) { sibling = list_entry(next->prev, struct vmap_area, list); if (sibling->va_end == va->va_start) { /* * If both neighbors are coalesced, it is important * to unlink the "next" node first, followed by merging * with "previous" one. Otherwise the tree might not be * fully populated if a sibling's augmented value is * "normalized" because of rotation operations. */ if (merged) __unlink_va(va, root, augment);
sibling->va_end = va->va_end;
/* Free vmap_area object. */ kmem_cache_free(vmap_area_cachep, va);
/* Point to the new merged area. */ va = sibling; merged = true; } }
insert: if (!merged) __link_va(va, root, parent, link, head, augment);
return va;}
static __always_inline struct vmap_area *merge_or_add_vmap_area(struct vmap_area *va, struct rb_root *root, struct list_head *head){ return __merge_or_add_vmap_area(va, root, head, false);}
static __always_inline struct vmap_area *merge_or_add_vmap_area_augment(struct vmap_area *va, struct rb_root *root, struct list_head *head){ va = __merge_or_add_vmap_area(va, root, head, true); if (va) augment_tree_propagate_from(va);
return va;}
static __always_inline boolis_within_this_va(struct vmap_area *va, unsigned long size, unsigned long align, unsigned long vstart){ unsigned long nva_start_addr;
if (va->va_start > vstart) nva_start_addr = ALIGN(va->va_start, align); else nva_start_addr = ALIGN(vstart, align);
/* Can be overflowed due to big size or alignment. */ if (nva_start_addr + size < nva_start_addr || nva_start_addr < vstart) return false;
return (nva_start_addr + size <= va->va_end);}
/* * Find the first free block(lowest start address) in the tree, * that will accomplish the request corresponding to passing * parameters. Please note, with an alignment bigger than PAGE_SIZE, * a search length is adjusted to account for worst case alignment * overhead. */static __always_inline struct vmap_area *find_vmap_lowest_match(struct rb_root *root, unsigned long size, unsigned long align, unsigned long vstart, bool adjust_search_size){ struct vmap_area *va; struct rb_node *node; unsigned long length;
/* Start from the root. */ node = root->rb_node;
/* Adjust the search size for alignment overhead. */ length = adjust_search_size ? size + align - 1 : size;
while (node) { va = rb_entry(node, struct vmap_area, rb_node);
if (get_subtree_max_size(node->rb_left) >= length && vstart < va->va_start) { node = node->rb_left; } else { if (is_within_this_va(va, size, align, vstart)) return va;
/* * Does not make sense to go deeper towards the right * sub-tree if it does not have a free block that is * equal or bigger to the requested search length. */ if (get_subtree_max_size(node->rb_right) >= length) { node = node->rb_right; continue; }
/* * OK. We roll back and find the first right sub-tree, * that will satisfy the search criteria. It can happen * due to "vstart" restriction or an alignment overhead * that is bigger then PAGE_SIZE. */ while ((node = rb_parent(node))) { va = rb_entry(node, struct vmap_area, rb_node); if (is_within_this_va(va, size, align, vstart)) return va;
if (get_subtree_max_size(node->rb_right) >= length && vstart <= va->va_start) { /* * Shift the vstart forward. Please note, we update it with * parent's start address adding "1" because we do not want * to enter same sub-tree after it has already been checked * and no suitable free block found there. */ vstart = va->va_start + 1; node = node->rb_right; break; } } } }
return NULL;}
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK#include <linux/random.h>
static struct vmap_area *find_vmap_lowest_linear_match(struct list_head *head, unsigned long size, unsigned long align, unsigned long vstart){ struct vmap_area *va;
list_for_each_entry(va, head, list) { if (!is_within_this_va(va, size, align, vstart)) continue;
return va; }
return NULL;}
static voidfind_vmap_lowest_match_check(struct rb_root *root, struct list_head *head, unsigned long size, unsigned long align){ struct vmap_area *va_1, *va_2; unsigned long vstart; unsigned int rnd;
get_random_bytes(&rnd, sizeof(rnd)); vstart = VMALLOC_START + rnd;
va_1 = find_vmap_lowest_match(root, size, align, vstart, false); va_2 = find_vmap_lowest_linear_match(head, size, align, vstart);
if (va_1 != va_2) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", va_1, va_2, vstart);}#endif
enum fit_type { NOTHING_FIT = 0, FL_FIT_TYPE = 1, /* full fit */ LE_FIT_TYPE = 2, /* left edge fit */ RE_FIT_TYPE = 3, /* right edge fit */ NE_FIT_TYPE = 4 /* no edge fit */};
static __always_inline enum fit_typeclassify_va_fit_type(struct vmap_area *va, unsigned long nva_start_addr, unsigned long size){ enum fit_type type;
/* Check if it is within VA. */ if (nva_start_addr < va->va_start || nva_start_addr + size > va->va_end) return NOTHING_FIT;
/* Now classify. */ if (va->va_start == nva_start_addr) { if (va->va_end == nva_start_addr + size) type = FL_FIT_TYPE; else type = LE_FIT_TYPE; } else if (va->va_end == nva_start_addr + size) { type = RE_FIT_TYPE; } else { type = NE_FIT_TYPE; }
return type;}
static __always_inline intva_clip(struct rb_root *root, struct list_head *head, struct vmap_area *va, unsigned long nva_start_addr, unsigned long size){ struct vmap_area *lva = NULL; enum fit_type type = classify_va_fit_type(va, nva_start_addr, size);
if (type == FL_FIT_TYPE) { /* * No need to split VA, it fully fits. * * | | * V NVA V * |---------------| */ unlink_va_augment(va, root); kmem_cache_free(vmap_area_cachep, va); } else if (type == LE_FIT_TYPE) { /* * Split left edge of fit VA. * * | | * V NVA V R * |-------|-------| */ va->va_start += size; } else if (type == RE_FIT_TYPE) { /* * Split right edge of fit VA. * * | | * L V NVA V * |-------|-------| */ va->va_end = nva_start_addr; } else if (type == NE_FIT_TYPE) { /* * Split no edge of fit VA. * * | | * L V NVA V R * |---|-------|---| */ lva = __this_cpu_xchg(ne_fit_preload_node, NULL); if (unlikely(!lva)) { /* * For percpu allocator we do not do any pre-allocation * and leave it as it is. The reason is it most likely * never ends up with NE_FIT_TYPE splitting. In case of * percpu allocations offsets and sizes are aligned to * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE * are its main fitting cases. * * There are a few exceptions though, as an example it is * a first allocation (early boot up) when we have "one" * big free space that has to be split. * * Also we can hit this path in case of regular "vmap" * allocations, if "this" current CPU was not preloaded. * See the comment in alloc_vmap_area() why. If so, then * GFP_NOWAIT is used instead to get an extra object for * split purpose. That is rare and most time does not * occur. * * What happens if an allocation gets failed. Basically, * an "overflow" path is triggered to purge lazily freed * areas to free some memory, then, the "retry" path is * triggered to repeat one more time. See more details * in alloc_vmap_area() function. */ lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); if (!lva) return -ENOMEM; }
/* * Build the remainder. */ lva->va_start = va->va_start; lva->va_end = nva_start_addr;
/* * Shrink this VA to remaining size. */ va->va_start = nva_start_addr + size; } else { return -EINVAL; }
if (type != FL_FIT_TYPE) { augment_tree_propagate_from(va);
if (lva) /* type == NE_FIT_TYPE */ insert_vmap_area_augment(lva, &va->rb_node, root, head); }
return 0;}
static unsigned longva_alloc(struct vmap_area *va, struct rb_root *root, struct list_head *head, unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend){ unsigned long nva_start_addr; int ret;
if (va->va_start > vstart) nva_start_addr = ALIGN(va->va_start, align); else nva_start_addr = ALIGN(vstart, align);
/* Check the "vend" restriction. */ if (nva_start_addr + size > vend) return -ERANGE;
/* Update the free vmap_area. */ ret = va_clip(root, head, va, nva_start_addr, size); if (ret) { WARN_ON_ONCE(ret != -ENOMEM); return ret; }
return nva_start_addr;}
/* * Returns a start address of the newly allocated area, if success. * Otherwise an error value is returned that indicates failure. */static __always_inline unsigned long__alloc_vmap_area(struct rb_root *root, struct list_head *head, unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend){ bool adjust_search_size = true; unsigned long nva_start_addr; struct vmap_area *va;
/* * Do not adjust when: * a) align <= PAGE_SIZE, because it does not make any sense. * All blocks(their start addresses) are at least PAGE_SIZE * aligned anyway; * b) a short range where a requested size corresponds to exactly * specified [vstart:vend] interval and an alignment > PAGE_SIZE. * With adjusted search length an allocation would not succeed. */ if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size)) adjust_search_size = false;
va = find_vmap_lowest_match(root, size, align, vstart, adjust_search_size); if (unlikely(!va)) return -ENOENT;
nva_start_addr = va_alloc(va, root, head, size, align, vstart, vend);
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK if (!IS_ERR_VALUE(nva_start_addr)) find_vmap_lowest_match_check(root, head, size, align);#endif
return nva_start_addr;}
/* * Free a region of KVA allocated by alloc_vmap_area */static void free_vmap_area(struct vmap_area *va){ struct vmap_node *vn = addr_to_node(va->va_start);
/* * Remove from the busy tree/list. */ spin_lock(&vn->busy.lock); unlink_va(va, &vn->busy.root); spin_unlock(&vn->busy.lock);
/* * Insert/Merge it back to the free tree/list. */ spin_lock(&free_vmap_area_lock); merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); spin_unlock(&free_vmap_area_lock);}
static inline voidpreload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node){ struct vmap_area *va = NULL, *tmp;
/* * Preload this CPU with one extra vmap_area object. It is used * when fit type of free area is NE_FIT_TYPE. It is best effort * pre-loading. If it fails va_clip() may return -ENOMEM from its * GFP_NOWAIT fallback. */ if (!this_cpu_read(ne_fit_preload_node)) va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
spin_lock(lock);
tmp = NULL; if (va && !__this_cpu_try_cmpxchg(ne_fit_preload_node, &tmp, va)) kmem_cache_free(vmap_area_cachep, va);}
static struct vmap_pool *size_to_va_pool(struct vmap_node *vn, unsigned long size){ unsigned int idx = (size - 1) / PAGE_SIZE;
if (idx < MAX_VA_SIZE_PAGES) return &vn->pool[idx];
return NULL;}
static boolnode_pool_add_va(struct vmap_node *n, struct vmap_area *va){ struct vmap_pool *vp;
vp = size_to_va_pool(n, va_size(va)); if (!vp) return false;
spin_lock(&n->pool_lock); list_add(&va->list, &vp->head); WRITE_ONCE(vp->len, vp->len + 1); spin_unlock(&n->pool_lock);
return true;}
static struct vmap_area *node_pool_del_va(struct vmap_node *vn, unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend){ struct vmap_area *va = NULL; struct vmap_pool *vp; int err = 0;
vp = size_to_va_pool(vn, size); if (!vp || list_empty(&vp->head)) return NULL;
spin_lock(&vn->pool_lock); if (!list_empty(&vp->head)) { va = list_first_entry(&vp->head, struct vmap_area, list);
if (IS_ALIGNED(va->va_start, align)) { /* * Do some sanity check and emit a warning * if one of below checks detects an error. */ err |= (va_size(va) != size); err |= (va->va_start < vstart); err |= (va->va_end > vend);
if (!WARN_ON_ONCE(err)) { list_del_init(&va->list); WRITE_ONCE(vp->len, vp->len - 1); } else { va = NULL; } } else { list_move_tail(&va->list, &vp->head); va = NULL; } } spin_unlock(&vn->pool_lock);
return va;}
static struct vmap_area *node_alloc(unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend, unsigned long *addr, unsigned int *vn_id){ struct vmap_area *va;
*vn_id = 0; *addr = -EINVAL;
/* * Fallback to a global heap if not vmalloc or there * is only one node. */ if (vstart != VMALLOC_START || vend != VMALLOC_END || nr_vmap_nodes == 1) return NULL;
*vn_id = raw_smp_processor_id() % nr_vmap_nodes; va = node_pool_del_va(id_to_node(*vn_id), size, align, vstart, vend); *vn_id = encode_vn_id(*vn_id);
if (va) *addr = va->va_start;
return va;}
static inline void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, unsigned long flags, const void *caller){ vm->flags = flags; vm->addr = (void *)va->va_start; vm->size = vm->requested_size = va_size(va); vm->caller = caller; va->vm = vm;}
/* * Allocate a region of KVA of the specified size and alignment, within the * vstart and vend. If vm is passed in, the two will also be bound. */static struct vmap_area *alloc_vmap_area(unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend, int node, gfp_t gfp_mask, unsigned long va_flags, struct vm_struct *vm){ struct vmap_node *vn; struct vmap_area *va; unsigned long freed; unsigned long addr; unsigned int vn_id; bool allow_block; int purged = 0; int ret;
if (unlikely(!size || offset_in_page(size) || !is_power_of_2(align))) return ERR_PTR(-EINVAL);
if (unlikely(!vmap_initialized)) return ERR_PTR(-EBUSY);
/* Only reclaim behaviour flags are relevant. */ gfp_mask = gfp_mask & GFP_RECLAIM_MASK; allow_block = gfpflags_allow_blocking(gfp_mask); might_sleep_if(allow_block);
/* * If a VA is obtained from a global heap(if it fails here) * it is anyway marked with this "vn_id" so it is returned * to this pool's node later. Such way gives a possibility * to populate pools based on users demand. * * On success a ready to go VA is returned. */ va = node_alloc(size, align, vstart, vend, &addr, &vn_id); if (!va) { va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); if (unlikely(!va)) return ERR_PTR(-ENOMEM);
/* * Only scan the relevant parts containing pointers to other objects * to avoid false negatives. */ kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); }
retry: if (IS_ERR_VALUE(addr)) { preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); addr = __alloc_vmap_area(&free_vmap_area_root, &free_vmap_area_list, size, align, vstart, vend); spin_unlock(&free_vmap_area_lock);
/* * This is not a fast path. Check if yielding is needed. This * is the only reschedule point in the vmalloc() path. */ if (allow_block) cond_resched(); }
trace_alloc_vmap_area(addr, size, align, vstart, vend, IS_ERR_VALUE(addr));
/* * If an allocation fails, the error value is * returned. Therefore trigger the overflow path. */ if (IS_ERR_VALUE(addr)) { if (allow_block) goto overflow;
/* * We can not trigger any reclaim logic because * sleeping is not allowed, thus fail an allocation. */ goto out_free_va; }
va->va_start = addr; va->va_end = addr + size; va->vm = NULL; va->flags = (va_flags | vn_id);
if (vm) { vm->addr = (void *)va->va_start; vm->size = va_size(va); va->vm = vm; }
vn = addr_to_node(va->va_start);
spin_lock(&vn->busy.lock); insert_vmap_area(va, &vn->busy.root, &vn->busy.head); spin_unlock(&vn->busy.lock);
BUG_ON(!IS_ALIGNED(va->va_start, align)); BUG_ON(va->va_start < vstart); BUG_ON(va->va_end > vend);
ret = kasan_populate_vmalloc(addr, size, gfp_mask); if (ret) { free_vmap_area(va); return ERR_PTR(ret); }
return va;
overflow: if (!purged) { reclaim_and_purge_vmap_areas(); purged = 1; goto retry; }
freed = 0; blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
if (freed > 0) { purged = 0; goto retry; }
if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) pr_warn("vmalloc_node_range for size %lu failed: Address range restricted to %#lx - %#lx\n", size, vstart, vend);
out_free_va: kmem_cache_free(vmap_area_cachep, va); return ERR_PTR(-EBUSY);}
int register_vmap_purge_notifier(struct notifier_block *nb){ return blocking_notifier_chain_register(&vmap_notify_list, nb);}EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
int unregister_vmap_purge_notifier(struct notifier_block *nb){ return blocking_notifier_chain_unregister(&vmap_notify_list, nb);}EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
/* * lazy_max_pages is the maximum amount of virtual address space we gather up * before attempting to purge with a TLB flush. * * There is a tradeoff here: a larger number will cover more kernel page tables * and take slightly longer to purge, but it will linearly reduce the number of * global TLB flushes that must be performed. It would seem natural to scale * this number up linearly with the number of CPUs (because vmapping activity * could also scale linearly with the number of CPUs), however it is likely * that in practice, workloads might be constrained in other ways that mean * vmap activity will not scale linearly with CPUs. Also, I want to be * conservative and not introduce a big latency on huge systems, so go with * a less aggressive log scale. It will still be an improvement over the old * code, and it will be simple to change the scale factor if we find that it * becomes a problem on bigger systems. */static unsigned long lazy_max_pages(void){ unsigned int log;
log = fls(num_online_cpus());
return log * (32UL * 1024 * 1024 / PAGE_SIZE);}
/* * Serialize vmap purging. There is no actual critical section protected * by this lock, but we want to avoid concurrent calls for performance * reasons and to make the pcpu_get_vm_areas more deterministic. */static DEFINE_MUTEX(vmap_purge_lock);
/* for per-CPU blocks */static void purge_fragmented_blocks_allcpus(void);
static voidreclaim_list_global(struct list_head *head){ struct vmap_area *va, *n;
if (list_empty(head)) return;
spin_lock(&free_vmap_area_lock); list_for_each_entry_safe(va, n, head, list) merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); spin_unlock(&free_vmap_area_lock);}
static voiddecay_va_pool_node(struct vmap_node *vn, bool full_decay){ LIST_HEAD(decay_list); struct rb_root decay_root = RB_ROOT; struct vmap_area *va, *nva; unsigned long n_decay, pool_len; int i;
for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { LIST_HEAD(tmp_list);
if (list_empty(&vn->pool[i].head)) continue;
/* Detach the pool, so no-one can access it. */ spin_lock(&vn->pool_lock); list_replace_init(&vn->pool[i].head, &tmp_list); spin_unlock(&vn->pool_lock);
pool_len = n_decay = vn->pool[i].len; WRITE_ONCE(vn->pool[i].len, 0);
/* Decay a pool by ~25% out of left objects. */ if (!full_decay) n_decay >>= 2; pool_len -= n_decay;
list_for_each_entry_safe(va, nva, &tmp_list, list) { if (!n_decay--) break;
list_del_init(&va->list); merge_or_add_vmap_area(va, &decay_root, &decay_list); }
/* * Attach the pool back if it has been partly decayed. * Please note, it is supposed that nobody(other contexts) * can populate the pool therefore a simple list replace * operation takes place here. */ if (!list_empty(&tmp_list)) { spin_lock(&vn->pool_lock); list_replace_init(&tmp_list, &vn->pool[i].head); WRITE_ONCE(vn->pool[i].len, pool_len); spin_unlock(&vn->pool_lock); } }
reclaim_list_global(&decay_list);}
#define KASAN_RELEASE_BATCH_SIZE 32
static voidkasan_release_vmalloc_node(struct vmap_node *vn){ struct vmap_area *va; unsigned long start, end; unsigned int batch_count = 0;
start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start; end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end;
list_for_each_entry(va, &vn->purge_list, list) { if (is_vmalloc_or_module_addr((void *) va->va_start)) kasan_release_vmalloc(va->va_start, va->va_end, va->va_start, va->va_end, KASAN_VMALLOC_PAGE_RANGE);
if (need_resched() || (++batch_count >= KASAN_RELEASE_BATCH_SIZE)) { cond_resched(); batch_count = 0; } }
kasan_release_vmalloc(start, end, start, end, KASAN_VMALLOC_TLB_FLUSH);}
static void purge_vmap_node(struct work_struct *work){ struct vmap_node *vn = container_of(work, struct vmap_node, purge_work); unsigned long nr_purged_pages = 0; struct vmap_area *va, *n_va; LIST_HEAD(local_list);
if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) kasan_release_vmalloc_node(vn);
vn->nr_purged = 0;
list_for_each_entry_safe(va, n_va, &vn->purge_list, list) { unsigned long nr = va_size(va) >> PAGE_SHIFT; unsigned int vn_id = decode_vn_id(va->flags);
list_del_init(&va->list);
nr_purged_pages += nr; vn->nr_purged++;
if (is_vn_id_valid(vn_id) && !vn->skip_populate) if (node_pool_add_va(vn, va)) continue;
/* Go back to global. */ list_add(&va->list, &local_list); }
atomic_long_sub(nr_purged_pages, &vmap_lazy_nr);
reclaim_list_global(&local_list);}
/* * Purges all lazily-freed vmap areas. */static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, bool full_pool_decay){ unsigned long nr_purged_areas = 0; unsigned int nr_purge_helpers; static cpumask_t purge_nodes; unsigned int nr_purge_nodes; struct vmap_node *vn; int i;
lockdep_assert_held(&vmap_purge_lock);
/* * Use cpumask to mark which node has to be processed. */ purge_nodes = CPU_MASK_NONE;
for_each_vmap_node(vn) { INIT_LIST_HEAD(&vn->purge_list); vn->skip_populate = full_pool_decay; decay_va_pool_node(vn, full_pool_decay);
if (RB_EMPTY_ROOT(&vn->lazy.root)) continue;
spin_lock(&vn->lazy.lock); WRITE_ONCE(vn->lazy.root.rb_node, NULL); list_replace_init(&vn->lazy.head, &vn->purge_list); spin_unlock(&vn->lazy.lock);
start = min(start, list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start);
end = max(end, list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end);
cpumask_set_cpu(node_to_id(vn), &purge_nodes); }
nr_purge_nodes = cpumask_weight(&purge_nodes); if (nr_purge_nodes > 0) { flush_tlb_kernel_range(start, end);
/* One extra worker is per a lazy_max_pages() full set minus one. */ nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages(); nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1;
for_each_cpu(i, &purge_nodes) { vn = &vmap_nodes[i];
if (nr_purge_helpers > 0) { INIT_WORK(&vn->purge_work, purge_vmap_node);
if (cpumask_test_cpu(i, cpu_online_mask)) schedule_work_on(i, &vn->purge_work); else schedule_work(&vn->purge_work);
nr_purge_helpers--; } else { vn->purge_work.func = NULL; purge_vmap_node(&vn->purge_work); nr_purged_areas += vn->nr_purged; } }
for_each_cpu(i, &purge_nodes) { vn = &vmap_nodes[i];
if (vn->purge_work.func) { flush_work(&vn->purge_work); nr_purged_areas += vn->nr_purged; } } }
trace_purge_vmap_area_lazy(start, end, nr_purged_areas); return nr_purged_areas > 0;}
/* * Reclaim vmap areas by purging fragmented blocks and purge_vmap_area_list. */static void reclaim_and_purge_vmap_areas(void)
{ mutex_lock(&vmap_purge_lock); purge_fragmented_blocks_allcpus(); __purge_vmap_area_lazy(ULONG_MAX, 0, true); mutex_unlock(&vmap_purge_lock);}
static void drain_vmap_area_work(struct work_struct *work){ mutex_lock(&vmap_purge_lock); __purge_vmap_area_lazy(ULONG_MAX, 0, false); mutex_unlock(&vmap_purge_lock);}
/* * Free a vmap area, caller ensuring that the area has been unmapped, * unlinked and flush_cache_vunmap had been called for the correct * range previously. */static void free_vmap_area_noflush(struct vmap_area *va){ unsigned long nr_lazy_max = lazy_max_pages(); unsigned long va_start = va->va_start; unsigned int vn_id = decode_vn_id(va->flags); struct vmap_node *vn; unsigned long nr_lazy;
if (WARN_ON_ONCE(!list_empty(&va->list))) return;
nr_lazy = atomic_long_add_return_relaxed(va_size(va) >> PAGE_SHIFT, &vmap_lazy_nr);
/* * If it was request by a certain node we would like to * return it to that node, i.e. its pool for later reuse. */ vn = is_vn_id_valid(vn_id) ? id_to_node(vn_id):addr_to_node(va->va_start);
spin_lock(&vn->lazy.lock); insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head); spin_unlock(&vn->lazy.lock);
trace_free_vmap_area_noflush(va_start, nr_lazy, nr_lazy_max);
/* After this point, we may free va at any time */ if (unlikely(nr_lazy > nr_lazy_max)) schedule_work(&drain_vmap_work);}
/* * Free and unmap a vmap area */static void free_unmap_vmap_area(struct vmap_area *va){ flush_cache_vunmap(va->va_start, va->va_end); vunmap_range_noflush(va->va_start, va->va_end); if (debug_pagealloc_enabled_static()) flush_tlb_kernel_range(va->va_start, va->va_end);
free_vmap_area_noflush(va);}
static inline int next_vmap_node_id(int i){ return (i + nr_vmap_nodes - 1) % nr_vmap_nodes;}
enum vmap_lock_mode { VMAP_LOCK, VMAP_TRYLOCK,};
/* * Search for a vmap_area at @addr across all vmap nodes. An * addr_to_node_id(addr) converts an address to a node index where * a VA is located. If VA spans several zones and passed addr is not * the same as va->va_start, what is not common, we may need to scan * extra nodes. See an example: * * <----va----> * -|-----|-----|-----|-----|- * 1 2 0 1 * * VA resides in node 1 whereas it spans 1, 2 an 0. If passed addr * is within 2 or 0 nodes we should do extra work. * * Returns the VA with @locked_vn->busy.lock held; the caller must * release it. If @mode is VMAP_TRYLOCK, nodes that cannot be locked * are skipped. */static struct vmap_area *find_vmap_area_lock(unsigned long addr, struct vmap_node **locked_vn, enum vmap_lock_mode mode){ struct vmap_node *vn; struct vmap_area *va; int i, j;
*locked_vn = NULL;
if (unlikely(!vmap_initialized)) return NULL;
i = j = addr_to_node_id(addr); do { vn = &vmap_nodes[i];
if (mode == VMAP_LOCK) { spin_lock(&vn->busy.lock); } else { if (!spin_trylock(&vn->busy.lock)) continue; }
va = __find_vmap_area(addr, &vn->busy.root); if (va) { *locked_vn = vn; return va; }
spin_unlock(&vn->busy.lock); } while ((i = next_vmap_node_id(i)) != j);
return NULL;}
struct vmap_area *find_vmap_area(unsigned long addr){ struct vmap_node *vn; struct vmap_area *va;
va = find_vmap_area_lock(addr, &vn, VMAP_LOCK); if (va) spin_unlock(&vn->busy.lock);
return va;}
static struct vmap_area *find_unlink_vmap_area(unsigned long addr){ struct vmap_node *vn; struct vmap_area *va;
va = find_vmap_area_lock(addr, &vn, VMAP_LOCK); if (va) { unlink_va(va, &vn->busy.root); spin_unlock(&vn->busy.lock); }
return va;}
/*** Per cpu kva allocator ***/
/* * vmap space is limited especially on 32 bit architectures. Ensure there is * room for at least 16 percpu vmap blocks per CPU. *//* * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess * instead (we just need a rough idea) */#if BITS_PER_LONG == 32#define VMALLOC_SPACE (128UL*1024*1024)#else#define VMALLOC_SPACE (128UL*1024*1024*1024)#endif
#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */#define VMAP_BBMAP_BITS \ VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
/* * Purge threshold to prevent overeager purging of fragmented blocks for * regular operations: Purge if vb->free is less than 1/4 of the capacity. */#define VMAP_PURGE_THRESHOLD (VMAP_BBMAP_BITS / 4)
#define VMAP_RAM 0x1 /* indicates vm_map_ram area*/#define VMAP_BLOCK 0x2 /* mark out the vmap_block sub-type*/#define VMAP_FLAGS_MASK 0x3
struct vmap_block_queue { spinlock_t lock; struct list_head free;
/* * An xarray requires an extra memory dynamically to * be allocated. If it is an issue, we can use rb-tree * instead. */ struct xarray vmap_blocks;};
struct vmap_block { spinlock_t lock; struct vmap_area *va; unsigned long free, dirty; DECLARE_BITMAP(used_map, VMAP_BBMAP_BITS); unsigned long dirty_min, dirty_max; /*< dirty range */ struct list_head free_list; struct rcu_head rcu_head; struct list_head purge; unsigned int cpu;};
/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
/* * In order to fast access to any "vmap_block" associated with a * specific address, we use a hash. * * A per-cpu vmap_block_queue is used in both ways, to serialize * an access to free block chains among CPUs(alloc path) and it * also acts as a vmap_block hash(alloc/free paths). It means we * overload it, since we already have the per-cpu array which is * used as a hash table. When used as a hash a 'cpu' passed to * per_cpu() is not actually a CPU but rather a hash index. * * A hash function is addr_to_vb_xa() which hashes any address * to a specific index(in a hash) it belongs to. This then uses a * per_cpu() macro to access an array with generated index. * * An example: * * CPU_1 CPU_2 CPU_0 * | | | * V V V * 0 10 20 30 40 50 60 * |------|------|------|------|------|------|...<vmap address space> * CPU0 CPU1 CPU2 CPU0 CPU1 CPU2 * * - CPU_1 invokes vm_unmap_ram(6), 6 belongs to CPU0 zone, thus * it access: CPU0/INDEX0 -> vmap_blocks -> xa_lock; * * - CPU_2 invokes vm_unmap_ram(11), 11 belongs to CPU1 zone, thus * it access: CPU1/INDEX1 -> vmap_blocks -> xa_lock; * * - CPU_0 invokes vm_unmap_ram(20), 20 belongs to CPU2 zone, thus * it access: CPU2/INDEX2 -> vmap_blocks -> xa_lock. * * This technique almost always avoids lock contention on insert/remove, * however xarray spinlocks protect against any contention that remains. */static struct xarray *addr_to_vb_xa(unsigned long addr){ int index = (addr / VMAP_BLOCK_SIZE) % nr_cpu_ids;
/* * Please note, nr_cpu_ids points on a highest set * possible bit, i.e. we never invoke cpumask_next() * if an index points on it which is nr_cpu_ids - 1. */ if (!cpu_possible(index)) index = cpumask_next(index, cpu_possible_mask);
return &per_cpu(vmap_block_queue, index).vmap_blocks;}
/* * We should probably have a fallback mechanism to allocate virtual memory * out of partially filled vmap blocks. However vmap block sizing should be * fairly reasonable according to the vmalloc size, so it shouldn't be a * big problem. */
static unsigned long addr_to_vb_idx(unsigned long addr){ addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); addr /= VMAP_BLOCK_SIZE; return addr;}
static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off){ unsigned long addr;
addr = va_start + (pages_off << PAGE_SHIFT); BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); return (void *)addr;}
/** * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this * block. Of course pages number can't exceed VMAP_BBMAP_BITS * @order: how many 2^order pages should be occupied in newly allocated block * @gfp_mask: flags for the page level allocator * * Return: virtual address in a newly allocated block or ERR_PTR(-errno) */static void *new_vmap_block(unsigned int order, gfp_t gfp_mask){ struct vmap_block_queue *vbq; struct vmap_block *vb; struct vmap_area *va; struct xarray *xa; unsigned long vb_idx; int node, err; void *vaddr;
node = numa_node_id();
vb = kmalloc_node(sizeof(struct vmap_block), gfp_mask, node); if (unlikely(!vb)) return ERR_PTR(-ENOMEM);
va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, VMALLOC_START, VMALLOC_END, node, gfp_mask, VMAP_RAM|VMAP_BLOCK, NULL); if (IS_ERR(va)) { kfree(vb); return ERR_CAST(va); }
vaddr = vmap_block_vaddr(va->va_start, 0); spin_lock_init(&vb->lock); vb->va = va; /* At least something should be left free */ BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); bitmap_zero(vb->used_map, VMAP_BBMAP_BITS); vb->free = VMAP_BBMAP_BITS - (1UL << order); vb->dirty = 0; vb->dirty_min = VMAP_BBMAP_BITS; vb->dirty_max = 0; bitmap_set(vb->used_map, 0, (1UL << order)); INIT_LIST_HEAD(&vb->free_list); vb->cpu = raw_smp_processor_id();
xa = addr_to_vb_xa(va->va_start); vb_idx = addr_to_vb_idx(va->va_start); err = xa_insert(xa, vb_idx, vb, gfp_mask); if (err) { kfree(vb); free_vmap_area(va); return ERR_PTR(err); } /* * list_add_tail_rcu could happened in another core * rather than vb->cpu due to task migration, which * is safe as list_add_tail_rcu will ensure the list's * integrity together with list_for_each_rcu from read * side. */ vbq = per_cpu_ptr(&vmap_block_queue, vb->cpu); spin_lock(&vbq->lock); list_add_tail_rcu(&vb->free_list, &vbq->free); spin_unlock(&vbq->lock);
return vaddr;}
static void free_vmap_block(struct vmap_block *vb){ struct vmap_node *vn; struct vmap_block *tmp; struct xarray *xa;
xa = addr_to_vb_xa(vb->va->va_start); tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start)); BUG_ON(tmp != vb);
vn = addr_to_node(vb->va->va_start); spin_lock(&vn->busy.lock); unlink_va(vb->va, &vn->busy.root); spin_unlock(&vn->busy.lock);
free_vmap_area_noflush(vb->va); kfree_rcu(vb, rcu_head);}
static bool purge_fragmented_block(struct vmap_block *vb, struct list_head *purge_list, bool force_purge){ struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, vb->cpu);
if (vb->free + vb->dirty != VMAP_BBMAP_BITS || vb->dirty == VMAP_BBMAP_BITS) return false;
/* Don't overeagerly purge usable blocks unless requested */ if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD)) return false;
/* prevent further allocs after releasing lock */ WRITE_ONCE(vb->free, 0); /* prevent purging it again */ WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS); vb->dirty_min = 0; vb->dirty_max = VMAP_BBMAP_BITS; spin_lock(&vbq->lock); list_del_rcu(&vb->free_list); spin_unlock(&vbq->lock); list_add_tail(&vb->purge, purge_list); return true;}
static void free_purged_blocks(struct list_head *purge_list){ struct vmap_block *vb, *n_vb;
list_for_each_entry_safe(vb, n_vb, purge_list, purge) { list_del(&vb->purge); free_vmap_block(vb); }}
static void purge_fragmented_blocks(int cpu){ LIST_HEAD(purge); struct vmap_block *vb; struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
rcu_read_lock(); list_for_each_entry_rcu(vb, &vbq->free, free_list) { unsigned long free = READ_ONCE(vb->free); unsigned long dirty = READ_ONCE(vb->dirty);
if (free + dirty != VMAP_BBMAP_BITS || dirty == VMAP_BBMAP_BITS) continue;
spin_lock(&vb->lock); purge_fragmented_block(vb, &purge, true); spin_unlock(&vb->lock); } rcu_read_unlock(); free_purged_blocks(&purge);}
static void purge_fragmented_blocks_allcpus(void){ int cpu;
for_each_possible_cpu(cpu) purge_fragmented_blocks(cpu);}
static void *vb_alloc(unsigned long size, gfp_t gfp_mask){ struct vmap_block_queue *vbq; struct vmap_block *vb; void *vaddr = NULL; unsigned int order;
BUG_ON(offset_in_page(size)); BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); if (WARN_ON(size == 0)) { /* * Allocating 0 bytes isn't what caller wants since * get_order(0) returns funny result. Just warn and terminate * early. */ return ERR_PTR(-EINVAL); } order = get_order(size);
rcu_read_lock(); vbq = raw_cpu_ptr(&vmap_block_queue); list_for_each_entry_rcu(vb, &vbq->free, free_list) { unsigned long pages_off;
if (READ_ONCE(vb->free) < (1UL << order)) continue;
spin_lock(&vb->lock); if (vb->free < (1UL << order)) { spin_unlock(&vb->lock); continue; }
pages_off = VMAP_BBMAP_BITS - vb->free; vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); WRITE_ONCE(vb->free, vb->free - (1UL << order)); bitmap_set(vb->used_map, pages_off, (1UL << order)); if (vb->free == 0) { spin_lock(&vbq->lock); list_del_rcu(&vb->free_list); spin_unlock(&vbq->lock); }
spin_unlock(&vb->lock); break; }
rcu_read_unlock();
/* Allocate new block if nothing was found */ if (!vaddr) vaddr = new_vmap_block(order, gfp_mask);
return vaddr;}
static void vb_free(unsigned long addr, unsigned long size){ unsigned long offset; unsigned int order; struct vmap_block *vb; struct xarray *xa;
BUG_ON(offset_in_page(size)); BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
flush_cache_vunmap(addr, addr + size);
order = get_order(size); offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT;
xa = addr_to_vb_xa(addr); vb = xa_load(xa, addr_to_vb_idx(addr));
spin_lock(&vb->lock); bitmap_clear(vb->used_map, offset, (1UL << order)); spin_unlock(&vb->lock);
vunmap_range_noflush(addr, addr + size);
if (debug_pagealloc_enabled_static()) flush_tlb_kernel_range(addr, addr + size);
spin_lock(&vb->lock);
/* Expand the not yet TLB flushed dirty range */ vb->dirty_min = min(vb->dirty_min, offset); vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order)); if (vb->dirty == VMAP_BBMAP_BITS) { BUG_ON(vb->free); spin_unlock(&vb->lock); free_vmap_block(vb); } else spin_unlock(&vb->lock);}
static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush){ LIST_HEAD(purge_list); int cpu;
if (unlikely(!vmap_initialized)) return;
mutex_lock(&vmap_purge_lock);
for_each_possible_cpu(cpu) { struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); struct vmap_block *vb; unsigned long idx;
rcu_read_lock(); xa_for_each(&vbq->vmap_blocks, idx, vb) { spin_lock(&vb->lock);
/* * Try to purge a fragmented block first. If it's * not purgeable, check whether there is dirty * space to be flushed. */ if (!purge_fragmented_block(vb, &purge_list, false) && vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) { unsigned long va_start = vb->va->va_start; unsigned long s, e;
s = va_start + (vb->dirty_min << PAGE_SHIFT); e = va_start + (vb->dirty_max << PAGE_SHIFT);
start = min(s, start); end = max(e, end);
/* Prevent that this is flushed again */ vb->dirty_min = VMAP_BBMAP_BITS; vb->dirty_max = 0;
flush = 1; } spin_unlock(&vb->lock); } rcu_read_unlock(); } free_purged_blocks(&purge_list);
if (!__purge_vmap_area_lazy(start, end, false) && flush) flush_tlb_kernel_range(start, end); mutex_unlock(&vmap_purge_lock);}
/** * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer * * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily * to amortize TLB flushing overheads. What this means is that any page you * have now, may, in a former life, have been mapped into kernel virtual * address by the vmap layer and so there might be some CPUs with TLB entries * still referencing that page (additional to the regular 1:1 kernel mapping). * * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can * be sure that none of the pages we have control over will have any aliases * from the vmap layer. */void vm_unmap_aliases(void){ _vm_unmap_aliases(ULONG_MAX, 0, 0);}EXPORT_SYMBOL_GPL(vm_unmap_aliases);
/** * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram * @mem: the pointer returned by vm_map_ram * @count: the count passed to that vm_map_ram call (cannot unmap partial) */void vm_unmap_ram(const void *mem, unsigned int count){ unsigned long size = (unsigned long)count << PAGE_SHIFT; unsigned long addr = (unsigned long)kasan_reset_tag(mem); struct vmap_area *va;
might_sleep(); BUG_ON(!addr); BUG_ON(addr < VMALLOC_START); BUG_ON(addr > VMALLOC_END); BUG_ON(!PAGE_ALIGNED(addr));
kasan_poison_vmalloc(mem, size);
if (likely(count <= VMAP_MAX_ALLOC)) { debug_check_no_locks_freed(mem, size); vb_free(addr, size); return; }
va = find_unlink_vmap_area(addr); if (WARN_ON_ONCE(!va)) return;
debug_check_no_locks_freed((void *)va->va_start, va_size(va)); free_unmap_vmap_area(va);}EXPORT_SYMBOL(vm_unmap_ram);
/** * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) * @pages: an array of pointers to the pages to be mapped * @count: number of pages * @node: prefer to allocate data structures on this node * * If you use this function for less than VMAP_MAX_ALLOC pages, it could be * faster than vmap so it's good. But if you mix long-life and short-life * objects with vm_map_ram(), it could consume lots of address space through * fragmentation (especially on a 32bit machine). You could see failures in * the end. Please use this function for short-lived objects. * * Returns: a pointer to the address that has been mapped, or %NULL on failure */void *vm_map_ram(struct page **pages, unsigned int count, int node){ unsigned long size = (unsigned long)count << PAGE_SHIFT; unsigned long addr; void *mem;
if (likely(count <= VMAP_MAX_ALLOC)) { mem = vb_alloc(size, GFP_KERNEL); if (IS_ERR(mem)) return NULL; addr = (unsigned long)mem; } else { struct vmap_area *va; va = alloc_vmap_area(size, PAGE_SIZE, VMALLOC_START, VMALLOC_END, node, GFP_KERNEL, VMAP_RAM, NULL); if (IS_ERR(va)) return NULL;
addr = va->va_start; mem = (void *)addr; }
if (vmap_pages_range(addr, addr + size, PAGE_KERNEL, pages, PAGE_SHIFT) < 0) { vm_unmap_ram(mem, count); return NULL; }
/* * Mark the pages as accessible, now that they are mapped. * With hardware tag-based KASAN, marking is skipped for * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). */ mem = kasan_unpoison_vmalloc(mem, size, KASAN_VMALLOC_PROT_NORMAL);
return mem;}EXPORT_SYMBOL(vm_map_ram);
static struct vm_struct *vmlist __initdata;
static inline unsigned int vm_area_page_order(const struct vm_struct *vm){#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC return vm->page_order;#else return 0;#endif}
unsigned int get_vm_area_page_order(const struct vm_struct *vm){ return vm_area_page_order(vm);}
static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order){#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC vm->page_order = order;#else BUG_ON(order != 0);#endif}
/** * vm_area_add_early - add vmap area early during boot * @vm: vm_struct to add * * This function is used to add fixed kernel vm area to vmlist before * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags * should contain proper values and the other fields should be zero. * * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. */void __init vm_area_add_early(struct vm_struct *vm){ struct vm_struct *tmp, **p;
BUG_ON(vmap_initialized); for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { if (tmp->addr >= vm->addr) { BUG_ON(tmp->addr < vm->addr + vm->size); break; } else BUG_ON(tmp->addr + tmp->size > vm->addr); } vm->next = *p; *p = vm;}
/** * vm_area_register_early - register vmap area early during boot * @vm: vm_struct to register * @align: requested alignment * * This function is used to register kernel vm area before * vmalloc_init() is called. @vm->size and @vm->flags should contain * proper values on entry and other fields should be zero. On return, * vm->addr contains the allocated address. * * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. */void __init vm_area_register_early(struct vm_struct *vm, size_t align){ unsigned long addr = ALIGN(VMALLOC_START, align); struct vm_struct *cur, **p;
BUG_ON(vmap_initialized);
for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) { if ((unsigned long)cur->addr - addr >= vm->size) break; addr = ALIGN((unsigned long)cur->addr + cur->size, align); }
BUG_ON(addr > VMALLOC_END - vm->size); vm->addr = (void *)addr; vm->next = *p; *p = vm; kasan_populate_early_vm_area_shadow(vm->addr, vm->size);}
void clear_vm_uninitialized_flag(struct vm_struct *vm){ /* * Before removing VM_UNINITIALIZED, * we should make sure that vm has proper values. * Pair with smp_rmb() in vread_iter() and vmalloc_info_show(). */ smp_wmb(); vm->flags &= ~VM_UNINITIALIZED;}
struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long align, unsigned long shift, unsigned long flags, unsigned long start, unsigned long end, int node, gfp_t gfp_mask, const void *caller){ struct vmap_area *va; struct vm_struct *area; unsigned long requested_size = size;
BUG_ON(in_nmi() || in_hardirq()); size = ALIGN(size, 1ul << shift); if (unlikely(!size)) return NULL;
if (flags & VM_IOREMAP) align = 1ul << clamp_t(int, get_count_order_long(size), PAGE_SHIFT, IOREMAP_MAX_ORDER);
area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); if (unlikely(!area)) return NULL;
if (!(flags & VM_NO_GUARD)) size += PAGE_SIZE;
area->flags = flags; area->caller = caller; area->requested_size = requested_size;
va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area); if (IS_ERR(va)) { kfree(area); return NULL; }
/* * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a * best-effort approach, as they can be mapped outside of vmalloc code. * For VM_ALLOC mappings, the pages are marked as accessible after * getting mapped in __vmalloc_node_range(). * With hardware tag-based KASAN, marking is skipped for * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). */ if (!(flags & VM_ALLOC)) area->addr = kasan_unpoison_vmalloc(area->addr, requested_size, KASAN_VMALLOC_PROT_NORMAL);
return area;}
struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, unsigned long start, unsigned long end, const void *caller){ return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, NUMA_NO_NODE, GFP_KERNEL, caller);}
/** * get_vm_area - reserve a contiguous kernel virtual area * @size: size of the area * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC * * Search an area of @size in the kernel virtual mapping area, * and reserved it for out purposes. Returns the area descriptor * on success or %NULL on failure. * * Return: the area descriptor on success or %NULL on failure. */struct vm_struct *get_vm_area(unsigned long size, unsigned long flags){ return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0));}
struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, const void *caller){ return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, caller);}
/** * find_vm_area - find a continuous kernel virtual area * @addr: base address * * Search for the kernel VM area starting at @addr, and return it. * It is up to the caller to do all required locking to keep the returned * pointer valid. * * Return: the area descriptor on success or %NULL on failure. */struct vm_struct *find_vm_area(const void *addr){ struct vmap_area *va;
va = find_vmap_area((unsigned long)addr); if (!va) return NULL;
return va->vm;}
/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address * * Search for the kernel VM area starting at @addr, and remove it. * This function returns the found VM area, but using it is NOT safe * on SMP machines, except for its size or flags. * * Return: the area descriptor on success or %NULL on failure. */struct vm_struct *remove_vm_area(const void *addr){ struct vmap_area *va; struct vm_struct *vm;
might_sleep();
if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", addr)) return NULL;
va = find_unlink_vmap_area((unsigned long)addr); if (!va || !va->vm) return NULL; vm = va->vm;
debug_check_no_locks_freed(vm->addr, get_vm_area_size(vm)); debug_check_no_obj_freed(vm->addr, get_vm_area_size(vm)); kasan_free_module_shadow(vm); kasan_poison_vmalloc(vm->addr, get_vm_area_size(vm));
free_unmap_vmap_area(va); return vm;}
static inline void set_area_direct_map(const struct vm_struct *area, int (*set_direct_map)(struct page *page, unsigned int nr)){ unsigned int nr = (1U << vm_area_page_order(area));
for (unsigned long i = 0; i < area->nr_pages; i += nr) { if (page_address(area->pages[i])) { int err = set_direct_map(area->pages[i], nr);
WARN_ON_ONCE(err); } }}
/* * Flush the vm mapping and reset the direct map. */static void vm_reset_perms(struct vm_struct *area){ unsigned long start = ULONG_MAX, end = 0; unsigned int page_order = vm_area_page_order(area); int flush_dmap = 0; unsigned long i;
/* * Find the start and end range of the direct mappings to make sure that * the vm_unmap_aliases() flush includes the direct map. */ for (i = 0; i < area->nr_pages; i += 1U << page_order) { unsigned long addr = (unsigned long)page_address(area->pages[i]);
if (addr) { unsigned long page_size;
page_size = PAGE_SIZE << page_order; start = min(addr, start); end = max(addr + page_size, end); flush_dmap = 1; } }
/* * Set direct map to something invalid so that it won't be cached if * there are any accesses after the TLB flush, then flush the TLB and * reset the direct map permissions to the default. */ set_area_direct_map(area, set_direct_map_invalid_noflush); _vm_unmap_aliases(start, end, flush_dmap); set_area_direct_map(area, set_direct_map_default_noflush);}
static void delayed_vfree_work(struct work_struct *w){ struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); struct llist_node *t, *llnode;
llist_for_each_safe(llnode, t, llist_del_all(&p->list)) vfree(llnode);}
/** * vfree_atomic - release memory allocated by vmalloc() * @addr: memory base address * * This one is just like vfree() but can be called in any atomic context * except NMIs. */void vfree_atomic(const void *addr){ struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred);
BUG_ON(in_nmi()); kmemleak_free(addr);
/* * Use raw_cpu_ptr() because this can be called from preemptible * context. Preemption is absolutely fine here, because the llist_add() * implementation is lockless, so it works even if we are adding to * another cpu's list. schedule_work() should be fine with this too. */ if (addr && llist_add((struct llist_node *)addr, &p->list)) schedule_work(&p->wq);}
/* * vm_area_free_pages - free a range of pages from a vmalloc allocation * @vm: the vm_struct containing the pages * @start_idx: first page index to free (inclusive) * @end_idx: last page index to free (exclusive) * * Free pages [start_idx, end_idx) updating NR_VMALLOC stat accounting. * Freed vm->pages[] entries are set to NULL. * Caller is responsible for unmapping (vunmap_range) and KASAN * poisoning before calling this. */static void vm_area_free_pages(struct vm_struct *vm, unsigned long start_idx, unsigned long end_idx){ unsigned long i;
if (!(vm->flags & VM_MAP_PUT_PAGES)) { for (i = start_idx; i < end_idx; i++) mod_lruvec_page_state(vm->pages[i], NR_VMALLOC, -1); } free_pages_bulk(vm->pages + start_idx, end_idx - start_idx);
for (i = start_idx; i < end_idx; i++) vm->pages[i] = NULL;}
/** * vfree - Release memory allocated by vmalloc() * @addr: Memory base address * * Free the virtually continuous memory area starting at @addr, as obtained * from one of the vmalloc() family of APIs. This will usually also free the * physical memory underlying the virtual allocation, but that memory is * reference counted, so it will not be freed until the last user goes away. * * If @addr is NULL, no operation is performed. * * Context: * May sleep if called *not* from interrupt context. * Must not be called in NMI context (strictly speaking, it could be * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling * conventions for vfree() arch-dependent would be a really bad idea). */void vfree(const void *addr){ struct vm_struct *vm;
if (unlikely(in_interrupt())) { vfree_atomic(addr); return; }
BUG_ON(in_nmi()); kmemleak_free(addr); might_sleep();
if (!addr) return;
vm = remove_vm_area(addr); if (unlikely(!vm)) { WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr); return; }
if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS)) vm_reset_perms(vm);
vm_area_free_pages(vm, 0, vm->nr_pages); kvfree(vm->pages); kfree(vm);}EXPORT_SYMBOL(vfree);
/** * vunmap - release virtual mapping obtained by vmap() * @addr: memory base address * * Free the virtually contiguous memory area starting at @addr, * which was created from the page array passed to vmap(). * * Must not be called in interrupt context. */void vunmap(const void *addr){ struct vm_struct *vm;
BUG_ON(in_interrupt()); might_sleep();
if (!addr) return; vm = remove_vm_area(addr); if (unlikely(!vm)) { WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n", addr); return; } kfree(vm);}EXPORT_SYMBOL(vunmap);
/** * vmap - map an array of pages into virtually contiguous space * @pages: array of page pointers * @count: number of pages to map * @flags: vm_area->flags * @prot: page protection for the mapping * * Maps @count pages from @pages into contiguous kernel virtual space. * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself * (which must be kmalloc or vmalloc memory) and one reference per pages in it * are transferred from the caller to vmap(), and will be freed / dropped when * vfree() is called on the return value. * * Return: the address of the area or %NULL on failure */void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot){ struct vm_struct *area; unsigned long addr; unsigned long size; /* In bytes */
might_sleep();
if (WARN_ON_ONCE(flags & VM_FLUSH_RESET_PERMS)) return NULL;
/* * Your top guard is someone else's bottom guard. Not having a top * guard compromises someone else's mappings too. */ if (WARN_ON_ONCE(flags & VM_NO_GUARD)) flags &= ~VM_NO_GUARD;
if (count > totalram_pages()) return NULL;
size = (unsigned long)count << PAGE_SHIFT; area = get_vm_area_caller(size, flags, __builtin_return_address(0)); if (!area) return NULL;
addr = (unsigned long)area->addr; if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), pages, PAGE_SHIFT) < 0) { vunmap(area->addr); return NULL; }
if (flags & VM_MAP_PUT_PAGES) { area->pages = pages; area->nr_pages = count; } return area->addr;}EXPORT_SYMBOL(vmap);
#ifdef CONFIG_VMAP_PFNstruct vmap_pfn_data { unsigned long *pfns; pgprot_t prot; unsigned int idx;};
static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private){ struct vmap_pfn_data *data = private; unsigned long pfn = data->pfns[data->idx]; pte_t ptent;
if (WARN_ON_ONCE(pfn_valid(pfn))) return -EINVAL;
ptent = pte_mkspecial(pfn_pte(pfn, data->prot)); set_pte_at(&init_mm, addr, pte, ptent);
data->idx++; return 0;}
/** * vmap_pfn - map an array of PFNs into virtually contiguous space * @pfns: array of PFNs * @count: number of pages to map * @prot: page protection for the mapping * * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns * the start address of the mapping. */void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot){ struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) }; struct vm_struct *area;
area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP, __builtin_return_address(0)); if (!area) return NULL; if (apply_to_page_range(&init_mm, (unsigned long)area->addr, count * PAGE_SIZE, vmap_pfn_apply, &data)) { free_vm_area(area); return NULL; }
flush_cache_vmap((unsigned long)area->addr, (unsigned long)area->addr + count * PAGE_SIZE);
return area->addr;}EXPORT_SYMBOL_GPL(vmap_pfn);#endif /* CONFIG_VMAP_PFN */
/* * Helper for vmalloc to adjust the gfp flags for certain allocations. */static inline gfp_t vmalloc_gfp_adjust(gfp_t flags, const bool large){ flags |= __GFP_NOWARN; if (large) flags &= ~__GFP_NOFAIL; return flags;}
static inline unsigned longvm_area_alloc_pages(gfp_t gfp, int nid, unsigned int order, unsigned long nr_pages, struct page **pages){ unsigned long nr_allocated = 0; unsigned long nr_remaining = nr_pages; unsigned int max_attempt_order = MAX_PAGE_ORDER; struct page *page; int i; unsigned int large_order = ilog2(nr_remaining); gfp_t large_gfp = vmalloc_gfp_adjust(gfp, large_order) & ~__GFP_DIRECT_RECLAIM;
large_order = min(max_attempt_order, large_order);
/* * Initially, attempt to have the page allocator give us large order * pages. Do not attempt allocating smaller than order chunks since * __vmap_pages_range() expects physically contiguous pages of exactly * order long chunks. */ while (large_order > order && nr_remaining) { if (nid == NUMA_NO_NODE) page = alloc_pages_noprof(large_gfp, large_order); else page = alloc_pages_node_noprof(nid, large_gfp, large_order);
if (unlikely(!page)) { max_attempt_order = --large_order; continue; }
mod_lruvec_page_state(page, NR_VMALLOC, 1 << large_order);
split_page(page, large_order); for (i = 0; i < (1U << large_order); i++) pages[nr_allocated + i] = page + i;
nr_allocated += 1U << large_order; nr_remaining = nr_pages - nr_allocated;
large_order = ilog2(nr_remaining); large_order = min(max_attempt_order, large_order); }
/* * For order-0 pages we make use of bulk allocator, if * the page array is partly or not at all populated due * to fails, fallback to a single page allocator that is * more permissive. */ if (!order) { while (nr_allocated < nr_pages) { unsigned int nr, nr_pages_request; unsigned long i;
/* * A maximum allowed request is hard-coded and is 100 * pages per call. That is done in order to prevent a * long preemption off scenario in the bulk-allocator * so the range is [1:100]. */ nr_pages_request = min(100UL, nr_pages - nr_allocated);
/* memory allocation should consider mempolicy, we can't * wrongly use nearest node when nid == NUMA_NO_NODE, * otherwise memory may be allocated in only one node, * but mempolicy wants to alloc memory by interleaving. */ if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) nr = alloc_pages_bulk_mempolicy_noprof(gfp, nr_pages_request, pages + nr_allocated); else nr = alloc_pages_bulk_node_noprof(gfp, nid, nr_pages_request, pages + nr_allocated);
for (i = nr_allocated; i < nr_allocated + nr; i++) mod_lruvec_page_state(pages[i], NR_VMALLOC, 1);
nr_allocated += nr;
/* * If zero or pages were obtained partly, * fallback to a single page allocator. */ if (nr != nr_pages_request) break; } }
/* High-order pages or fallback path if "bulk" fails. */ while (nr_allocated < nr_pages) { if (!(gfp & __GFP_NOFAIL) && fatal_signal_pending(current)) break;
if (nid == NUMA_NO_NODE) page = alloc_pages_noprof(gfp, order); else page = alloc_pages_node_noprof(nid, gfp, order);
if (unlikely(!page)) break;
mod_lruvec_page_state(page, NR_VMALLOC, 1 << order);
/* * High-order allocations must be able to be treated as * independent small pages by callers (as they can with * small-page vmallocs). Some drivers do their own refcounting * on vmalloc_to_page() pages, some use page->mapping, * page->lru, etc. */ if (order) split_page(page, order);
/* * Careful, we allocate and map page-order pages, but * tracking is done per PAGE_SIZE page so as to keep the * vm_struct APIs independent of the physical/mapped size. */ for (i = 0; i < (1U << order); i++) pages[nr_allocated + i] = page + i;
nr_allocated += 1U << order; }
return nr_allocated;}
static LLIST_HEAD(pending_vm_area_cleanup);static void cleanup_vm_area_work(struct work_struct *work){ struct vm_struct *area, *tmp; struct llist_node *head;
head = llist_del_all(&pending_vm_area_cleanup); if (!head) return;
llist_for_each_entry_safe(area, tmp, head, llnode) { if (!area->pages) free_vm_area(area); else vfree(area->addr); }}
/* * Helper for __vmalloc_area_node() to defer cleanup * of partially initialized vm_struct in error paths. */static DECLARE_WORK(cleanup_vm_area, cleanup_vm_area_work);static void defer_vm_area_cleanup(struct vm_struct *area){ if (llist_add(&area->llnode, &pending_vm_area_cleanup)) schedule_work(&cleanup_vm_area);}
/* * Page tables allocations ignore external GFP. Enforces it by * the memalloc scope API. It is used by vmalloc internals and * KASAN shadow population only. * * GFP to scope mapping: * * non-blocking (no __GFP_DIRECT_RECLAIM) - memalloc_noreclaim_save() * GFP_NOFS - memalloc_nofs_save() * GFP_NOIO - memalloc_noio_save() * __GFP_RETRY_MAYFAIL, __GFP_NORETRY - memalloc_noreclaim_save() * to prevent OOMs * * Returns a flag cookie to pair with restore. */unsigned intmemalloc_apply_gfp_scope(gfp_t gfp_mask){ unsigned int flags = 0;
if (!gfpflags_allow_blocking(gfp_mask) || (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_NORETRY))) flags = memalloc_noreclaim_save(); else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) flags = memalloc_nofs_save(); else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) flags = memalloc_noio_save();
/* 0 - no scope applied. */ return flags;}
voidmemalloc_restore_scope(unsigned int flags){ if (flags) memalloc_flags_restore(flags);}
static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot, unsigned int page_shift, int node){ const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; bool nofail = gfp_mask & __GFP_NOFAIL; unsigned long addr = (unsigned long)area->addr; unsigned long size = get_vm_area_size(area); unsigned long array_size; unsigned long nr_small_pages = size >> PAGE_SHIFT; unsigned int page_order = page_shift - PAGE_SHIFT; unsigned int flags; int ret;
array_size = nr_small_pages * sizeof(struct page *);
/* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */ if (!gfpflags_allow_blocking(gfp_mask)) nofail = false;
if (!(gfp_mask & (GFP_DMA | GFP_DMA32))) gfp_mask |= __GFP_HIGHMEM;
/* Please note that the recursion is strictly bounded. */ if (array_size > PAGE_SIZE) { area->pages = __vmalloc_node_noprof(array_size, 1, nested_gfp, node, area->caller); } else { area->pages = kmalloc_node_noprof(array_size, nested_gfp, node); }
if (!area->pages) { warn_alloc(gfp_mask, NULL, "vmalloc error: size %lu, failed to allocated page array size %lu", nr_small_pages * PAGE_SIZE, array_size); goto fail; }
/* * High-order nofail allocations are really expensive and * potentially dangerous (pre-mature OOM, disruptive reclaim * and compaction etc. * * Please note, the __vmalloc_node_range_noprof() falls-back * to order-0 pages if high-order attempt is unsuccessful. */ area->nr_pages = vm_area_alloc_pages( vmalloc_gfp_adjust(gfp_mask, page_order), node, page_order, nr_small_pages, area->pages);
/* * If not enough pages were obtained to accomplish an * allocation request, free them via vfree() if any. */ if (area->nr_pages != nr_small_pages) { /* * vm_area_alloc_pages() can fail due to insufficient memory but * also:- * * - a pending fatal signal * - insufficient huge page-order pages * * Since we always retry allocations at order-0 in the huge page * case a warning for either is spurious. */ if (!fatal_signal_pending(current) && page_order == 0) warn_alloc(gfp_mask, NULL, "vmalloc error: size %lu, failed to allocate pages", nr_small_pages * PAGE_SIZE); goto fail; }
/* * page tables allocations ignore external gfp mask, enforce it * by the scope API */ flags = memalloc_apply_gfp_scope(gfp_mask); do { ret = __vmap_pages_range(addr, addr + size, prot, area->pages, page_shift, nested_gfp); if (nofail && (ret < 0)) schedule_timeout_uninterruptible(1); } while (nofail && (ret < 0)); memalloc_restore_scope(flags);
if (ret < 0) { warn_alloc(gfp_mask, NULL, "vmalloc error: size %lu, failed to map pages", area->nr_pages * PAGE_SIZE); goto fail; }
set_vm_area_page_order(area, page_order); return area->addr;
fail: defer_vm_area_cleanup(area); return NULL;}
/* * See __vmalloc_node_range() for a clear list of supported vmalloc flags. * This gfp lists all flags currently passed through vmalloc. Currently, * __GFP_ZERO is used by BPF and __GFP_NORETRY is used by percpu. Both drm * and BPF also use GFP_USER. Additionally, various users pass * GFP_KERNEL_ACCOUNT. Xfs uses __GFP_NOLOCKDEP. */#define GFP_VMALLOC_SUPPORTED (GFP_KERNEL | GFP_ATOMIC | GFP_NOWAIT |\ __GFP_NOFAIL | __GFP_ZERO |\ __GFP_NORETRY | __GFP_RETRY_MAYFAIL |\ GFP_NOFS | GFP_NOIO | GFP_KERNEL_ACCOUNT |\ GFP_USER | __GFP_NOLOCKDEP | __GFP_SKIP_KASAN)
static gfp_t vmalloc_fix_flags(gfp_t flags){ gfp_t invalid_mask = flags & ~GFP_VMALLOC_SUPPORTED;
flags &= GFP_VMALLOC_SUPPORTED; WARN_ONCE(1, "Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n", invalid_mask, &invalid_mask, flags, &flags); return flags;}
/** * __vmalloc_node_range - allocate virtually contiguous memory * @size: allocation size * @align: desired alignment * @start: vm area range start * @end: vm area range end * @gfp_mask: flags for the page level allocator * @prot: protection mask for the allocated pages * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address * * Allocate enough pages to cover @size from the page level * allocator with @gfp_mask flags and map them into contiguous * virtual range with protection @prot. * * Supported GFP classes: %GFP_KERNEL, %GFP_ATOMIC, %GFP_NOWAIT, * %__GFP_RETRY_MAYFAIL, %__GFP_NORETRY, %GFP_NOFS and %GFP_NOIO. * Zone modifiers are not supported. * Please note %GFP_ATOMIC and %GFP_NOWAIT are supported only * by __vmalloc(). * * Retry modifiers: only %__GFP_NOFAIL is fully supported; * %__GFP_NORETRY and %__GFP_RETRY_MAYFAIL are supported with limitation, * i.e. page tables are allocated with NOWAIT semantic so they might fail * under moderate memory pressure. * * %__GFP_NOWARN can be used to suppress failure messages. * * %__GFP_SKIP_KASAN can be used to skip unpoisoning of mapped pages * (when prot=%PAGE_KERNEL). * * %VM_ALLOW_HUGE_VMAP allocates huge pages when possible and falls back to * base pages if huge page allocation fails. * * %VM_REQUIRE_HUGE_VMAP implies %VM_ALLOW_HUGE_VMAP and fails instead of * silently falling back to base pages. * * Can not be called from interrupt nor NMI contexts. * Return: the address of the area or %NULL on failure */void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller){ struct vm_struct *area; void *ret; kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE; unsigned long original_align = align; unsigned int shift = PAGE_SHIFT; bool skip_vmalloc_kasan = kasan_hw_tags_enabled() && (gfp_mask & __GFP_SKIP_KASAN);
if (WARN_ON_ONCE(!size)) return NULL;
if ((size >> PAGE_SHIFT) > totalram_pages()) { warn_alloc(gfp_mask, NULL, "vmalloc error: size %lu, exceeds total pages", size); return NULL; }
/* VM_REQUIRE_HUGE_VMAP implies VM_ALLOW_HUGE_VMAP */ if (vm_flags & VM_REQUIRE_HUGE_VMAP) vm_flags |= VM_ALLOW_HUGE_VMAP;
if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { /* * Try huge pages. Only try for PAGE_KERNEL allocations, * others like modules don't yet expect huge pages in * their allocations due to apply_to_page_range not * supporting them. */
if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) shift = PMD_SHIFT; else shift = arch_vmap_pte_supported_shift(size);
align = max(original_align, 1UL << shift); }
if ((vm_flags & VM_REQUIRE_HUGE_VMAP) && shift == PAGE_SHIFT) return NULL;
again: area = __get_vm_area_node(size, align, shift, VM_ALLOC | VM_UNINITIALIZED | vm_flags, start, end, node, gfp_mask & ~__GFP_SKIP_KASAN, caller); if (!area) { bool nofail = gfp_mask & __GFP_NOFAIL; warn_alloc(gfp_mask, NULL, "vmalloc error: size %lu, align 0x%lx, vm_struct allocation failed%s", size, align, (nofail) ? ". Retrying." : ""); if (nofail) { schedule_timeout_uninterruptible(1); goto again; } goto fail; }
/* * Prepare arguments for __vmalloc_area_node() and * kasan_unpoison_vmalloc(). */ if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) { if (kasan_hw_tags_enabled() && !skip_vmalloc_kasan) { /* * Modify protection bits to allow tagging. * This must be done before mapping. */ prot = arch_vmap_pgprot_tagged(prot);
/* * Skip page_alloc poisoning and zeroing for physical * pages backing VM_ALLOC mapping. Memory is instead * poisoned and zeroed by kasan_unpoison_vmalloc(). */ gfp_mask |= __GFP_SKIP_KASAN | __GFP_SKIP_ZERO; }
/* Take note that the mapping is PAGE_KERNEL. */ kasan_flags |= KASAN_VMALLOC_PROT_NORMAL; }
/* Allocate physical pages and map them into vmalloc space. */ ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node); if (!ret) goto fail;
/* * Mark the pages as accessible, now that they are mapped. * The condition for setting KASAN_VMALLOC_INIT should complement the * one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check * to make sure that memory is initialized under the same conditions. * Tag-based KASAN modes only assign tags to normal non-executable * allocations, see __kasan_unpoison_vmalloc(). */ kasan_flags |= KASAN_VMALLOC_VM_ALLOC; if (!want_init_on_free() && want_init_on_alloc(gfp_mask) && (gfp_mask & __GFP_SKIP_ZERO)) kasan_flags |= KASAN_VMALLOC_INIT; /* KASAN_VMALLOC_PROT_NORMAL already set if required. */ if (!skip_vmalloc_kasan) area->addr = kasan_unpoison_vmalloc(area->addr, size, kasan_flags);
/* * In this function, newly allocated vm_struct has VM_UNINITIALIZED * flag. It means that vm_struct is not fully initialized. * Now, it is fully initialized, so remove this flag here. */ clear_vm_uninitialized_flag(area);
if (!(vm_flags & VM_DEFER_KMEMLEAK)) kmemleak_vmalloc(area, PAGE_ALIGN(size), gfp_mask);
return area->addr;
fail: if (shift > PAGE_SHIFT && !(vm_flags & VM_REQUIRE_HUGE_VMAP)) { shift = PAGE_SHIFT; align = original_align; goto again; }
return NULL;}
/** * __vmalloc_node - allocate virtually contiguous memory * @size: allocation size * @align: desired alignment * @gfp_mask: flags for the page level allocator * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address * * Allocate enough pages to cover @size from the page level allocator with * @gfp_mask flags. Map them into contiguous kernel virtual space. * * Semantics of @gfp_mask (including reclaim/retry modifiers such as * __GFP_NOFAIL) are the same as in __vmalloc_node_range_noprof(). * * Return: pointer to the allocated memory or %NULL on error */void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_mask, int node, const void *caller){ return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, gfp_mask, PAGE_KERNEL, 0, node, caller);}/* * This is only for performance analysis of vmalloc and stress purpose. * It is required by vmalloc test module, therefore do not use it other * than that. */#ifdef CONFIG_TEST_VMALLOC_MODULEEXPORT_SYMBOL_GPL(__vmalloc_node_noprof);#endif
void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask){ if (unlikely(gfp_mask & ~GFP_VMALLOC_SUPPORTED)) gfp_mask = vmalloc_fix_flags(gfp_mask); return __vmalloc_node_noprof(size, 1, gfp_mask, NUMA_NO_NODE, __builtin_return_address(0));}EXPORT_SYMBOL(__vmalloc_noprof);
/** * vmalloc - allocate virtually contiguous memory * @size: allocation size * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. * * Return: pointer to the allocated memory or %NULL on error */void *vmalloc_noprof(unsigned long size){ return __vmalloc_node_noprof(size, 1, GFP_KERNEL, NUMA_NO_NODE, __builtin_return_address(0));}EXPORT_SYMBOL(vmalloc_noprof);
/** * vmalloc_huge_node - allocate virtually contiguous memory, allow huge pages * @size: allocation size * @gfp_mask: flags for the page level allocator * @node: node to use for allocation or NUMA_NO_NODE * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * If @size is greater than or equal to PMD_SIZE, allow using * huge pages for the memory * * Return: pointer to the allocated memory or %NULL on error */void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node){ if (unlikely(gfp_mask & ~GFP_VMALLOC_SUPPORTED)) gfp_mask = vmalloc_fix_flags(gfp_mask); return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0));}EXPORT_SYMBOL_GPL(vmalloc_huge_node_noprof);
/** * vzalloc - allocate virtually contiguous memory with zero fill * @size: allocation size * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * The memory allocated is set to zero. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. * * Return: pointer to the allocated memory or %NULL on error */void *vzalloc_noprof(unsigned long size){ return __vmalloc_node_noprof(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, __builtin_return_address(0));}EXPORT_SYMBOL(vzalloc_noprof);
/** * vmalloc_user - allocate zeroed virtually contiguous memory for userspace * @size: allocation size * * The resulting memory area is zeroed so it can be mapped to userspace * without leaking data. * * Return: pointer to the allocated memory or %NULL on error */void *vmalloc_user_noprof(unsigned long size){ return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, VM_USERMAP, NUMA_NO_NODE, __builtin_return_address(0));}EXPORT_SYMBOL(vmalloc_user_noprof);
/** * vmalloc_node - allocate memory on a specific node * @size: allocation size * @node: numa node * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. * * Return: pointer to the allocated memory or %NULL on error */void *vmalloc_node_noprof(unsigned long size, int node){ return __vmalloc_node_noprof(size, 1, GFP_KERNEL, node, __builtin_return_address(0));}EXPORT_SYMBOL(vmalloc_node_noprof);
/** * vzalloc_node - allocate memory on a specific node with zero fill * @size: allocation size * @node: numa node * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * The memory allocated is set to zero. * * Return: pointer to the allocated memory or %NULL on error */void *vzalloc_node_noprof(unsigned long size, int node){ return __vmalloc_node_noprof(size, 1, GFP_KERNEL | __GFP_ZERO, node, __builtin_return_address(0));}EXPORT_SYMBOL(vzalloc_node_noprof);
/** * vrealloc_node_align - reallocate virtually contiguous memory; contents * remain unchanged * @p: object to reallocate memory for * @size: the size to reallocate * @align: requested alignment * @flags: the flags for the page level allocator * @nid: node number of the target node * * If @p is %NULL, vrealloc_XXX() behaves exactly like vmalloc_XXX(). If @size * is 0 and @p is not a %NULL pointer, the object pointed to is freed. * * If the caller wants the new memory to be on specific node *only*, * __GFP_THISNODE flag should be set, otherwise the function will try to avoid * reallocation and possibly disregard the specified @nid. * * If __GFP_ZERO logic is requested, callers must ensure that, starting with the * initial memory allocation, every subsequent call to this API for the same * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that * __GFP_ZERO is not fully honored by this API. * * Requesting an alignment that is bigger than the alignment of the existing * allocation will fail. * * In any case, the contents of the object pointed to are preserved up to the * lesser of the new and old sizes. * * This function must not be called concurrently with itself or vfree() for the * same memory allocation. * * Return: pointer to the allocated memory; %NULL if @size is zero or in case of * failure */void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, gfp_t flags, int nid){ struct vm_struct *vm = NULL; size_t alloced_size = 0; size_t old_size = 0; void *n;
if (!size) { vfree(p); return NULL; }
if (p) { vm = find_vm_area(p); if (unlikely(!vm)) { WARN(1, "Trying to vrealloc() nonexistent vm area (%p)\n", p); return NULL; }
alloced_size = get_vm_area_size(vm); old_size = vm->requested_size; if (WARN(alloced_size < old_size, "vrealloc() has mismatched area vs requested sizes (%p)\n", p)) return NULL; if (WARN(!IS_ALIGNED((unsigned long)p, align), "will not reallocate with a bigger alignment (0x%lx)\n", align)) return NULL; if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && nid != page_to_nid(vmalloc_to_page(p))) goto need_realloc; } else { /* * If p is NULL, vrealloc behaves exactly like vmalloc. * Skip the shrink and in-place grow paths. */ goto need_realloc; }
if (size <= old_size) { unsigned long new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
/* Zero out "freed" memory, potentially for future realloc. */ if (want_init_on_free() || want_init_on_alloc(flags)) memset((void *)p + size, 0, old_size - size);
/* * Free tail pages when shrink crosses a page boundary. * * Skip huge page allocations (page_order > 0) as partial * freeing would require splitting. * * Skip VM_FLUSH_RESET_PERMS, as direct-map permissions must * be reset before pages are returned to the allocator. * * Skip VM_USERMAP, as remap_vmalloc_range_partial() validates * mapping requests against the unchanged vm->size; freeing * tail pages would cause vmalloc_to_page() to return NULL for * the unmapped range. * * Skip if either GFP_NOFS or GFP_NOIO are used. * kmemleak_free_part() internally allocates with * GFP_KERNEL, which could trigger a recursive deadlock * if we are under filesystem or I/O reclaim. */ if (new_nr_pages < vm->nr_pages && !vm_area_page_order(vm) && !(vm->flags & (VM_FLUSH_RESET_PERMS | VM_USERMAP)) && gfp_has_io_fs(flags)) { unsigned long addr = (unsigned long)kasan_reset_tag(p); unsigned long old_nr_pages = vm->nr_pages;
/* * Use the node lock to synchronize with concurrent * readers (vmalloc_info_show). */ struct vmap_node *vn = addr_to_node(addr);
spin_lock(&vn->busy.lock); vm->nr_pages = new_nr_pages; spin_unlock(&vn->busy.lock);
/* Notify kmemleak of the reduced allocation size before unmapping. */ kmemleak_free_part((void *)addr + (new_nr_pages << PAGE_SHIFT), (old_nr_pages - new_nr_pages) << PAGE_SHIFT);
vunmap_range(addr + (new_nr_pages << PAGE_SHIFT), addr + (old_nr_pages << PAGE_SHIFT));
vm_area_free_pages(vm, new_nr_pages, old_nr_pages); } vm->requested_size = size; kasan_vrealloc(p, old_size, size); return (void *)p; }
/* * We already have the bytes available in the allocation; use them. */ if (size <= vm->nr_pages << PAGE_SHIFT) { /* * No need to zero memory here, as unused memory will have * already been zeroed at initial allocation time or during * realloc shrink time. */ vm->requested_size = size; kasan_vrealloc(p, old_size, size); return (void *)p; }
need_realloc: /* TODO: Grow the vm_area, i.e. allocate and map additional pages. */ n = __vmalloc_node_noprof(size, align, flags, nid, __builtin_return_address(0));
if (!n) return NULL;
if (p) { memcpy(n, p, min(size, old_size)); vfree(p); }
return n;}EXPORT_SYMBOL(vrealloc_node_align_noprof);
#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)#define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)#else/* * 64b systems should always have either DMA or DMA32 zones. For others * GFP_DMA32 should do the right thing and use the normal zone. */#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)#endif
/** * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) * @size: allocation size * * Allocate enough 32bit PA addressable pages to cover @size from the * page level allocator and map them into contiguous kernel virtual space. * * Return: pointer to the allocated memory or %NULL on error */void *vmalloc_32_noprof(unsigned long size){ return __vmalloc_node_noprof(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, __builtin_return_address(0));}EXPORT_SYMBOL(vmalloc_32_noprof);
/** * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory * @size: allocation size * * The resulting memory area is 32bit addressable and zeroed so it can be * mapped to userspace without leaking data. * * Return: pointer to the allocated memory or %NULL on error */void *vmalloc_32_user_noprof(unsigned long size){ return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, VM_USERMAP, NUMA_NO_NODE, __builtin_return_address(0));}EXPORT_SYMBOL(vmalloc_32_user_noprof);
/* * Atomically zero bytes in the iterator. * * Returns the number of zeroed bytes. */static size_t zero_iter(struct iov_iter *iter, size_t count){ size_t remains = count;
while (remains > 0) { size_t num, copied;
num = min_t(size_t, remains, PAGE_SIZE); copied = copy_page_to_iter_nofault(ZERO_PAGE(0), 0, num, iter); remains -= copied;
if (copied < num) break; }
return count - remains;}
/* * small helper routine, copy contents to iter from addr. * If the page is not present, fill zero. * * Returns the number of copied bytes. */static size_t aligned_vread_iter(struct iov_iter *iter, const char *addr, size_t count){ size_t remains = count; struct page *page;
while (remains > 0) { unsigned long offset, length; size_t copied = 0;
offset = offset_in_page(addr); length = PAGE_SIZE - offset; if (length > remains) length = remains; page = vmalloc_to_page(addr); /* * To do safe access to this _mapped_ area, we need lock. But * adding lock here means that we need to add overhead of * vmalloc()/vfree() calls for this _debug_ interface, rarely * used. Instead of that, we'll use an local mapping via * copy_page_to_iter_nofault() and accept a small overhead in * this access function. */ if (page) copied = copy_page_to_iter_nofault(page, offset, length, iter); else copied = zero_iter(iter, length);
addr += copied; remains -= copied;
if (copied != length) break; }
return count - remains;}
/* * Read from a vm_map_ram region of memory. * * Returns the number of copied bytes. */static size_t vmap_ram_vread_iter(struct iov_iter *iter, const char *addr, size_t count, unsigned long flags){ char *start; struct vmap_block *vb; struct xarray *xa; unsigned long offset; unsigned int rs, re; size_t remains, n;
/* * If it's area created by vm_map_ram() interface directly, but * not further subdividing and delegating management to vmap_block, * handle it here. */ if (!(flags & VMAP_BLOCK)) return aligned_vread_iter(iter, addr, count);
remains = count;
/* * Area is split into regions and tracked with vmap_block, read out * each region and zero fill the hole between regions. */ xa = addr_to_vb_xa((unsigned long) addr); vb = xa_load(xa, addr_to_vb_idx((unsigned long)addr)); if (!vb) goto finished_zero;
spin_lock(&vb->lock); if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) { spin_unlock(&vb->lock); goto finished_zero; }
for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) { size_t copied;
if (remains == 0) goto finished;
start = vmap_block_vaddr(vb->va->va_start, rs);
if (addr < start) { size_t to_zero = min_t(size_t, start - addr, remains); size_t zeroed = zero_iter(iter, to_zero);
addr += zeroed; remains -= zeroed;
if (remains == 0 || zeroed != to_zero) goto finished; }
/*it could start reading from the middle of used region*/ offset = offset_in_page(addr); n = ((re - rs + 1) << PAGE_SHIFT) - offset; if (n > remains) n = remains;
copied = aligned_vread_iter(iter, start + offset, n);
addr += copied; remains -= copied;
if (copied != n) goto finished; }
spin_unlock(&vb->lock);
finished_zero: /* zero-fill the left dirty or free regions */ return count - remains + zero_iter(iter, remains);finished: /* We couldn't copy/zero everything */ spin_unlock(&vb->lock); return count - remains;}
/** * vread_iter() - read vmalloc area in a safe way to an iterator. * @iter: the iterator to which data should be written. * @addr: vm address. * @count: number of bytes to be read. * * This function checks that addr is a valid vmalloc'ed area, and * copies data from that area to a given iterator. If the given memory range * of [addr...addr+count) includes some valid address, data is copied to * proper area of @iter. If there are memory holes, they'll be zero-filled. * IOREMAP area is treated as memory hole and no copy is done. * * If [addr...addr+count) doesn't includes any intersects with alive * vm_struct area, returns 0. * * Note: In usual ops, vread_iter() is never necessary because the caller * should know vmalloc() area is valid and can use memcpy(). * This is for routines which have to access vmalloc area without * any information, as /proc/kcore. * * Return: number of bytes for which addr and iter should be advanced * (same number as @count) or %0 if [addr...addr+count) doesn't * include any intersection with valid vmalloc area */long vread_iter(struct iov_iter *iter, const char *addr, size_t count){ struct vmap_node *vn; struct vmap_area *va; struct vm_struct *vm; char *vaddr; size_t n, size, flags, remains; unsigned long next;
addr = kasan_reset_tag(addr);
/* Don't allow overflow */ if ((unsigned long) addr + count < count) count = -(unsigned long) addr;
remains = count;
vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va); if (!vn) goto finished_zero;
/* no intersects with alive vmap_area */ if ((unsigned long)addr + remains <= va->va_start) goto finished_zero;
do { size_t copied;
if (remains == 0) goto finished;
vm = va->vm; flags = va->flags & VMAP_FLAGS_MASK; /* * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need * be set together with VMAP_RAM. */ WARN_ON(flags == VMAP_BLOCK);
if (!vm && !flags) goto next_va;
if (vm && (vm->flags & VM_UNINITIALIZED)) goto next_va;
/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ smp_rmb();
vaddr = (char *) va->va_start; if (vm) /* * For VM_ALLOC areas, use nr_pages rather than * get_vm_area_size() because vrealloc() may shrink * the mapping without updating area->size. Other * mapping types (vmap, ioremap) don't set nr_pages. */ size = (vm->flags & VM_ALLOC && vm->nr_pages) ? (vm->nr_pages << PAGE_SHIFT) : get_vm_area_size(vm); else size = va_size(va);
if (addr >= vaddr + size) goto next_va;
if (addr < vaddr) { size_t to_zero = min_t(size_t, vaddr - addr, remains); size_t zeroed = zero_iter(iter, to_zero);
addr += zeroed; remains -= zeroed;
if (remains == 0 || zeroed != to_zero) goto finished; }
n = vaddr + size - addr; if (n > remains) n = remains;
if (flags & VMAP_RAM) copied = vmap_ram_vread_iter(iter, addr, n, flags); else if (!(vm && (vm->flags & (VM_IOREMAP | VM_SPARSE)))) copied = aligned_vread_iter(iter, addr, n); else /* IOREMAP | SPARSE area is treated as memory hole */ copied = zero_iter(iter, n);
addr += copied; remains -= copied;
if (copied != n) goto finished;
next_va: next = va->va_end; spin_unlock(&vn->busy.lock); } while ((vn = find_vmap_area_exceed_addr_lock(next, &va)));
finished_zero: if (vn) spin_unlock(&vn->busy.lock);
/* zero-fill memory holes */ return count - remains + zero_iter(iter, remains);finished: /* Nothing remains, or We couldn't copy/zero everything. */ if (vn) spin_unlock(&vn->busy.lock);
return count - remains;}
/** * remap_vmalloc_range_partial - map vmalloc pages to userspace * @vma: vma to cover * @uaddr: target user address to start at * @kaddr: virtual address of vmalloc kernel memory * @pgoff: offset from @kaddr to start at * @size: size of map area * * Returns: 0 for success, -Exxx on failure * * This function checks that @kaddr is a valid vmalloc'ed area, * and that it is big enough to cover the range starting at * @uaddr in @vma. Will return failure if that criteria isn't * met. * * Similar to remap_pfn_range() (see mm/memory.c) */int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, void *kaddr, unsigned long pgoff, unsigned long size){ struct vm_struct *area; unsigned long off; unsigned long end_index;
if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) return -EINVAL;
size = PAGE_ALIGN(size);
if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) return -EINVAL;
area = find_vm_area(kaddr); if (!area) return -EINVAL;
if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) return -EINVAL;
if (check_add_overflow(size, off, &end_index) || end_index > get_vm_area_size(area)) return -EINVAL; kaddr += off;
do { struct page *page = vmalloc_to_page(kaddr); int ret;
ret = vm_insert_page(vma, uaddr, page); if (ret) return ret;
uaddr += PAGE_SIZE; kaddr += PAGE_SIZE; size -= PAGE_SIZE; } while (size > 0);
vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
return 0;}
/** * remap_vmalloc_range - map vmalloc pages to userspace * @vma: vma to cover (map full range of vma) * @addr: vmalloc memory * @pgoff: number of pages into addr before first page to map * * Returns: 0 for success, -Exxx on failure * * This function checks that addr is a valid vmalloc'ed area, and * that it is big enough to cover the vma. Will return failure if * that criteria isn't met. * * Similar to remap_pfn_range() (see mm/memory.c) */int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, unsigned long pgoff){ return remap_vmalloc_range_partial(vma, vma->vm_start, addr, pgoff, vma->vm_end - vma->vm_start);}EXPORT_SYMBOL(remap_vmalloc_range);
void free_vm_area(struct vm_struct *area){ struct vm_struct *ret; ret = remove_vm_area(area->addr); BUG_ON(ret != area); kfree(area);}EXPORT_SYMBOL_GPL(free_vm_area);
#ifdef CONFIG_SMPstatic struct vmap_area *node_to_va(struct rb_node *n){ return rb_entry_safe(n, struct vmap_area, rb_node);}
/** * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to * @addr: target address * * Returns: vmap_area if it is found. If there is no such area * the first highest(reverse order) vmap_area is returned * i.e. va->va_start < addr && va->va_end < addr or NULL * if there are no any areas before @addr. */static struct vmap_area *pvm_find_va_enclose_addr(unsigned long addr){ struct vmap_area *va, *tmp; struct rb_node *n;
n = free_vmap_area_root.rb_node; va = NULL;
while (n) { tmp = rb_entry(n, struct vmap_area, rb_node); if (tmp->va_start <= addr) { va = tmp; if (tmp->va_end >= addr) break;
n = n->rb_right; } else { n = n->rb_left; } }
return va;}
/** * pvm_determine_end_from_reverse - find the highest aligned address * of free block below VMALLOC_END * @va: * in - the VA we start the search(reverse order); * out - the VA with the highest aligned end address. * @align: alignment for required highest address * * Returns: determined end address within vmap_area */static unsigned longpvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align){ unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); unsigned long addr;
if (likely(*va)) { list_for_each_entry_from_reverse((*va), &free_vmap_area_list, list) { addr = min((*va)->va_end & ~(align - 1), vmalloc_end); if ((*va)->va_start < addr) return addr; } }
return 0;}
/** * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator * @offsets: array containing offset of each area * @sizes: array containing size of each area * @nr_vms: the number of areas to allocate * @align: alignment, all entries in @offsets and @sizes must be aligned to this * @gfp: allocation flags passed to the underlying memory allocator * * Returns: kmalloc'd vm_struct pointer array pointing to allocated * vm_structs on success, %NULL on failure * * Percpu allocator wants to use congruent vm areas so that it can * maintain the offsets among percpu areas. This function allocates * congruent vmalloc areas for it. These areas tend to be scattered * pretty far, distance between two areas easily going up to gigabytes. * To avoid interacting with regular vmallocs, these areas are allocated * from top. * * Despite its complicated look, this allocator is rather simple. It * does everything top-down and scans free blocks from the end looking * for matching base. While scanning, if any of the areas do not fit the * base address is pulled down to fit the area. Scanning is repeated till * all the areas fit and then all necessary data structures are inserted * and the result is returned. */struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, size_t align, gfp_t gfp){ const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); struct vmap_area **vas, *va; struct vm_struct **vms; int area, area2, last_area, term_area; unsigned long base, start, size, end, last_end, orig_start, orig_end; bool purged = false;
/* verify parameters and allocate data structures */ BUG_ON(offset_in_page(align) || !is_power_of_2(align)); for (last_area = 0, area = 0; area < nr_vms; area++) { start = offsets[area]; end = start + sizes[area];
/* is everything aligned properly? */ BUG_ON(!IS_ALIGNED(offsets[area], align)); BUG_ON(!IS_ALIGNED(sizes[area], align));
/* detect the area with the highest address */ if (start > offsets[last_area]) last_area = area;
for (area2 = area + 1; area2 < nr_vms; area2++) { unsigned long start2 = offsets[area2]; unsigned long end2 = start2 + sizes[area2];
BUG_ON(start2 < end && start < end2); } } last_end = offsets[last_area] + sizes[last_area];
if (vmalloc_end - vmalloc_start < last_end) { WARN_ON(true); return NULL; }
vms = kzalloc_objs(vms[0], nr_vms, gfp); vas = kzalloc_objs(vas[0], nr_vms, gfp); if (!vas || !vms) goto err_free2;
for (area = 0; area < nr_vms; area++) { vas[area] = kmem_cache_zalloc(vmap_area_cachep, gfp); vms[area] = kzalloc_obj(struct vm_struct, gfp); if (!vas[area] || !vms[area]) goto err_free; }retry: spin_lock(&free_vmap_area_lock);
/* start scanning - we scan from the top, begin with the last area */ area = term_area = last_area; start = offsets[area]; end = start + sizes[area];
va = pvm_find_va_enclose_addr(vmalloc_end); base = pvm_determine_end_from_reverse(&va, align) - end;
while (true) { /* * base might have underflowed, add last_end before * comparing. */ if (base + last_end < vmalloc_start + last_end) goto overflow;
/* * Fitting base has not been found. */ if (va == NULL) goto overflow;
/* * If required width exceeds current VA block, move * base downwards and then recheck. */ if (base + end > va->va_end) { base = pvm_determine_end_from_reverse(&va, align) - end; term_area = area; continue; }
/* * If this VA does not fit, move base downwards and recheck. */ if (base + start < va->va_start) { va = node_to_va(rb_prev(&va->rb_node)); base = pvm_determine_end_from_reverse(&va, align) - end; term_area = area; continue; }
/* * This area fits, move on to the previous one. If * the previous one is the terminal one, we're done. */ area = (area + nr_vms - 1) % nr_vms; if (area == term_area) break;
start = offsets[area]; end = start + sizes[area]; va = pvm_find_va_enclose_addr(base + end); }
/* we've found a fitting base, insert all va's */ for (area = 0; area < nr_vms; area++) { int ret;
start = base + offsets[area]; size = sizes[area];
va = pvm_find_va_enclose_addr(start); if (WARN_ON_ONCE(va == NULL)) /* It is a BUG(), but trigger recovery instead. */ goto recovery;
ret = va_clip(&free_vmap_area_root, &free_vmap_area_list, va, start, size); if (unlikely(ret)) { /* * -ENOMEM from the GFP_NOWAIT fallback is expected. * Anything else is a BUG(), but trigger recovery instead. */ WARN_ON_ONCE(ret != -ENOMEM); goto recovery; }
/* Allocated area. */ va = vas[area]; va->va_start = start; va->va_end = start + size; }
spin_unlock(&free_vmap_area_lock);
/* populate the kasan shadow space */ for (area = 0; area < nr_vms; area++) { if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], gfp)) goto err_free_shadow; }
/* insert all vm's */ for (area = 0; area < nr_vms; area++) { struct vmap_node *vn = addr_to_node(vas[area]->va_start);
spin_lock(&vn->busy.lock); insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, pcpu_get_vm_areas); spin_unlock(&vn->busy.lock); }
/* * Mark allocated areas as accessible. Do it now as a best-effort * approach, as they can be mapped outside of vmalloc code. * With hardware tag-based KASAN, marking is skipped for * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). */ kasan_unpoison_vmap_areas(vms, nr_vms, KASAN_VMALLOC_PROT_NORMAL);
kfree(vas); return vms;
recovery: /* * Remove previously allocated areas. There is no * need in removing these areas from the busy tree, * because they are inserted only on the final step * and when pcpu_get_vm_areas() is success. */ while (area--) { orig_start = vas[area]->va_start; orig_end = vas[area]->va_end; va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, &free_vmap_area_list); if (va) kasan_release_vmalloc(orig_start, orig_end, va->va_start, va->va_end, KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH); vas[area] = NULL; }
overflow: spin_unlock(&free_vmap_area_lock); if (!purged) { reclaim_and_purge_vmap_areas(); purged = true;
/* Before "retry", check if we recover. */ for (area = 0; area < nr_vms; area++) { if (vas[area]) continue;
vas[area] = kmem_cache_zalloc( vmap_area_cachep, gfp); if (!vas[area]) goto err_free; }
goto retry; }
err_free: for (area = 0; area < nr_vms; area++) { if (vas[area]) kmem_cache_free(vmap_area_cachep, vas[area]);
kfree(vms[area]); }err_free2: kfree(vas); kfree(vms); return NULL;
err_free_shadow: spin_lock(&free_vmap_area_lock); /* * We release all the vmalloc shadows, even the ones for regions that * hadn't been successfully added. This relies on kasan_release_vmalloc * being able to tolerate this case. */ for (area = 0; area < nr_vms; area++) { orig_start = vas[area]->va_start; orig_end = vas[area]->va_end; va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, &free_vmap_area_list); if (va) kasan_release_vmalloc(orig_start, orig_end, va->va_start, va->va_end, KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH); vas[area] = NULL; kfree(vms[area]); } spin_unlock(&free_vmap_area_lock); goto err_free2;}
/** * pcpu_free_vm_areas - free vmalloc areas for percpu allocator * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() * @nr_vms: the number of allocated areas * * Free vm_structs and the array allocated by pcpu_get_vm_areas(). */void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms){ int i;
for (i = 0; i < nr_vms; i++) free_vm_area(vms[i]); kfree(vms);}#endif /* CONFIG_SMP */
#ifdef CONFIG_PRINTKbool vmalloc_dump_obj(void *object){ const void *caller; struct vm_struct *vm; struct vmap_area *va; struct vmap_node *vn; unsigned long addr; unsigned long nr_pages;
addr = PAGE_ALIGN_DOWN((unsigned long) object);
va = find_vmap_area_lock(addr, &vn, VMAP_TRYLOCK); if (!va || !va->vm) { if (va) spin_unlock(&vn->busy.lock); return false; }
vm = va->vm; addr = (unsigned long) vm->addr; caller = vm->caller; nr_pages = vm->nr_pages; spin_unlock(&vn->busy.lock);
pr_cont(" %lu-page vmalloc region starting at %#lx allocated at %pS\n", nr_pages, addr, caller);
return true;}#endif
#ifdef CONFIG_PROC_FS
/* * Print number of pages allocated on each memory node. * * This function can only be called if CONFIG_NUMA is enabled * and VM_UNINITIALIZED bit in v->flags is disabled. */static void show_numa_info(struct seq_file *m, struct vm_struct *v, unsigned int *counters){ unsigned int step = 1U << vm_area_page_order(v); unsigned long i; unsigned int nr;
if (!counters) return;
memset(counters, 0, nr_node_ids * sizeof(unsigned int));
for (i = 0; i < v->nr_pages; i += step) counters[page_to_nid(v->pages[i])] += step; for_each_node_state(nr, N_HIGH_MEMORY) if (counters[nr]) seq_printf(m, " N%u=%u", nr, counters[nr]);}
static void show_purge_info(struct seq_file *m){ struct vmap_node *vn; struct vmap_area *va;
for_each_vmap_node(vn) { spin_lock(&vn->lazy.lock); list_for_each_entry(va, &vn->lazy.head, list) { seq_printf(m, "0x%p-0x%p %7ld unpurged vm_area\n", (void *)va->va_start, (void *)va->va_end, va_size(va)); } spin_unlock(&vn->lazy.lock); }}
static void show_busy_info(struct seq_file *m){ struct vmap_node *vn; struct vmap_area *va; struct vm_struct *v; unsigned int *counters;
if (IS_ENABLED(CONFIG_NUMA)) counters = kmalloc_array(nr_node_ids, sizeof(unsigned int), GFP_KERNEL);
for_each_vmap_node(vn) { spin_lock(&vn->busy.lock); list_for_each_entry(va, &vn->busy.head, list) { if (!va->vm) { if (va->flags & VMAP_RAM) seq_printf(m, "0x%p-0x%p %7ld vm_map_ram\n", (void *)va->va_start, (void *)va->va_end, va_size(va));
continue; }
v = va->vm; if (v->flags & VM_UNINITIALIZED) continue;
/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ smp_rmb();
seq_printf(m, "0x%p-0x%p %7ld", v->addr, v->addr + v->size, v->size);
if (v->caller) seq_printf(m, " %pS", v->caller);
if (v->nr_pages) seq_printf(m, " pages=%lu", v->nr_pages);
if (v->phys_addr) seq_printf(m, " phys=%pa", &v->phys_addr);
if (v->flags & VM_IOREMAP) seq_puts(m, " ioremap");
if (v->flags & VM_SPARSE) seq_puts(m, " sparse");
if (v->flags & VM_ALLOC) seq_puts(m, " vmalloc");
if (v->flags & VM_MAP) seq_puts(m, " vmap");
if (v->flags & VM_USERMAP) seq_puts(m, " user");
if (v->flags & VM_DMA_COHERENT) seq_puts(m, " dma-coherent");
if (is_vmalloc_addr(v->pages)) seq_puts(m, " vpages");
if (IS_ENABLED(CONFIG_NUMA)) show_numa_info(m, v, counters);
seq_putc(m, '\n'); } spin_unlock(&vn->busy.lock); }
if (IS_ENABLED(CONFIG_NUMA)) kfree(counters);}
static int vmalloc_info_show(struct seq_file *m, void *p){ show_busy_info(m);
/* * As a final step, dump "unpurged" areas. */ show_purge_info(m); return 0;}
static int __init proc_vmalloc_init(void){ proc_create_single("vmallocinfo", 0400, NULL, vmalloc_info_show); return 0;}module_init(proc_vmalloc_init);
#endif
static void __init vmap_insert_free_area(unsigned long start, unsigned long end){ struct vmap_area *free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
if (!WARN_ON_ONCE(!free)) { free->va_start = start; free->va_end = end; insert_vmap_area_augment(free, NULL, &free_vmap_area_root, &free_vmap_area_list); }}
static void __init vmap_init_free_space(void){ unsigned long vmap_start = 1; const unsigned long vmap_end = ULONG_MAX; struct vm_struct *busy;
/* * B F B B B F * -|-----|.....|-----|-----|-----|.....|- * | The KVA space | * |<--------------------------------->| */ for (busy = vmlist; busy; busy = busy->next) { if ((unsigned long) busy->addr - vmap_start > 0) vmap_insert_free_area(vmap_start, (unsigned long) busy->addr);
vmap_start = (unsigned long) busy->addr + busy->size; }
if (vmap_end - vmap_start > 0) vmap_insert_free_area(vmap_start, vmap_end);}
static void vmap_init_nodes(void){ struct vmap_node *vn; int i;
#if BITS_PER_LONG == 64 /* * A high threshold of max nodes is fixed and bound to 128, * thus a scale factor is 1 for systems where number of cores * are less or equal to specified threshold. * * As for NUMA-aware notes. For bigger systems, for example * NUMA with multi-sockets, where we can end-up with thousands * of cores in total, a "sub-numa-clustering" should be added. * * In this case a NUMA domain is considered as a single entity * with dedicated sub-nodes in it which describe one group or * set of cores. Therefore a per-domain purging is supposed to * be added as well as a per-domain balancing. */ int n = clamp_t(unsigned int, num_possible_cpus(), 1, 128);
if (n > 1) { vn = kmalloc_objs(*vn, n, GFP_NOWAIT); if (vn) { /* Node partition is 16 pages. */ vmap_zone_size = (1 << 4) * PAGE_SIZE; nr_vmap_nodes = n; vmap_nodes = vn; } else { pr_err("Failed to allocate an array. Disable a node layer\n"); } }#endif
for_each_vmap_node(vn) { vn->busy.root = RB_ROOT; INIT_LIST_HEAD(&vn->busy.head); spin_lock_init(&vn->busy.lock);
vn->lazy.root = RB_ROOT; INIT_LIST_HEAD(&vn->lazy.head); spin_lock_init(&vn->lazy.lock);
for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { INIT_LIST_HEAD(&vn->pool[i].head); WRITE_ONCE(vn->pool[i].len, 0); }
spin_lock_init(&vn->pool_lock); }}
static unsigned longvmap_node_shrink_count(struct shrinker *shrink, struct shrink_control *sc){ unsigned long count = 0; struct vmap_node *vn; int i;
for_each_vmap_node(vn) { for (i = 0; i < MAX_VA_SIZE_PAGES; i++) count += READ_ONCE(vn->pool[i].len); }
return count ? count : SHRINK_EMPTY;}
static unsigned longvmap_node_shrink_scan(struct shrinker *shrink, struct shrink_control *sc){ struct vmap_node *vn;
guard(mutex)(&vmap_purge_lock); for_each_vmap_node(vn) decay_va_pool_node(vn, true);
return SHRINK_STOP;}
void __init vmalloc_init(void){ struct shrinker *vmap_node_shrinker; struct vmap_area *va; struct vmap_node *vn; struct vm_struct *tmp; int i;
/* * Create the cache for vmap_area objects. */ vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
for_each_possible_cpu(i) { struct vmap_block_queue *vbq; struct vfree_deferred *p;
vbq = &per_cpu(vmap_block_queue, i); spin_lock_init(&vbq->lock); INIT_LIST_HEAD(&vbq->free); xa_init(&vbq->vmap_blocks);
p = &per_cpu(vfree_deferred, i); init_llist_head(&p->list); INIT_WORK(&p->wq, delayed_vfree_work); }
/* * Setup nodes before importing vmlist. */ vmap_init_nodes();
/* Import existing vmlist entries. */ for (tmp = vmlist; tmp; tmp = tmp->next) { va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); if (WARN_ON_ONCE(!va)) continue;
va->va_start = (unsigned long)tmp->addr; va->va_end = va->va_start + tmp->size; va->vm = tmp;
vn = addr_to_node(va->va_start); insert_vmap_area(va, &vn->busy.root, &vn->busy.head); }
/* * Now we can initialize a free vmap space. */ vmap_init_free_space(); vmap_initialized = true;
vmap_node_shrinker = shrinker_alloc(0, "vmap-node"); if (!vmap_node_shrinker) { pr_err("Failed to allocate vmap-node shrinker!\n"); return; }
vmap_node_shrinker->count_objects = vmap_node_shrink_count; vmap_node_shrinker->scan_objects = vmap_node_shrink_scan; shrinker_register(vmap_node_shrinker);}