Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509// SPDX-License-Identifier: GPL-2.0-or-later/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * Kernel Memory Controller * Copyright (C) 2012 Parallels Inc. and Google Inc. * Authors: Glauber Costa and Suleiman Souhlal * * Native page reclaim * Charge lifetime sanitation * Lockless page tracking & accounting * Unified hierarchy configuration model * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner * * Per memcg lru locking * Copyright (C) 2020 Alibaba, Inc, Alex Shi */
#include <linux/cgroup-defs.h>#include <linux/page_counter.h>#include <linux/memcontrol.h>#include <linux/cgroup.h>#include <linux/cpuset.h>#include <linux/sched/mm.h>#include <linux/shmem_fs.h>#include <linux/hugetlb.h>#include <linux/pagemap.h>#include <linux/folio_batch.h>#include <linux/vm_event_item.h>#include <linux/smp.h>#include <linux/page-flags.h>#include <linux/backing-dev.h>#include <linux/bit_spinlock.h>#include <linux/rcupdate.h>#include <linux/limits.h>#include <linux/export.h>#include <linux/list.h>#include <linux/mutex.h>#include <linux/rbtree.h>#include <linux/slab.h>#include <linux/swapops.h>#include <linux/spinlock.h>#include <linux/fs.h>#include <linux/seq_file.h>#include <linux/vmpressure.h>#include <linux/memremap.h>#include <linux/mm_inline.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/lockdep.h>#include <linux/resume_user_mode.h>#include <linux/psi.h>#include <linux/seq_buf.h>#include <linux/sched/isolation.h>#include <linux/kmemleak.h>#include <linux/irq_work.h>#include "internal.h"#include "swap.h"#include "swap_table.h"#include <net/sock.h>#include <net/ip.h>#include "slab.h"#include "memcontrol-v1.h"
#include <linux/uaccess.h>
#define CREATE_TRACE_POINTS#include <trace/events/memcg.h>#undef CREATE_TRACE_POINTS
#include <trace/events/vmscan.h>
struct cgroup_subsys memory_cgrp_subsys __read_mostly;EXPORT_SYMBOL(memory_cgrp_subsys);
struct mem_cgroup *root_mem_cgroup __read_mostly;EXPORT_SYMBOL(root_mem_cgroup);
/* Active memory cgroup to use from an interrupt context */DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
/* Socket memory accounting disabled? */static bool cgroup_memory_nosocket __ro_after_init;
/* Kernel memory accounting disabled? */static bool cgroup_memory_nokmem __ro_after_init;
/* BPF memory accounting disabled? */static bool cgroup_memory_nobpf __ro_after_init;
static struct workqueue_struct *memcg_wq __ro_after_init;
static struct kmem_cache *memcg_cachep;static struct kmem_cache *memcg_pn_cachep;
#ifdef CONFIG_CGROUP_WRITEBACKstatic DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);#endif
static inline bool task_is_dying(void){ return tsk_is_oom_victim(current) || fatal_signal_pending(current) || (current->flags & PF_EXITING);}
/* Some nice accessors for the vmpressure. */struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg){ if (!memcg) memcg = root_mem_cgroup; return &memcg->vmpressure;}
struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr){ return container_of(vmpr, struct mem_cgroup, vmpressure);}
#define SEQ_BUF_SIZE SZ_4K#define CURRENT_OBJCG_UPDATE_BIT 0#define CURRENT_OBJCG_UPDATE_FLAG (1UL << CURRENT_OBJCG_UPDATE_BIT)
static DEFINE_SPINLOCK(objcg_lock);
bool mem_cgroup_kmem_disabled(void){ return cgroup_memory_nokmem;}
static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages);
static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages){ mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); if (!mem_cgroup_is_root(memcg)) memcg_uncharge(memcg, nr_pages);}
static void obj_cgroup_release(struct percpu_ref *ref){ struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); unsigned int nr_bytes; unsigned int nr_pages; unsigned long flags;
/* * At this point all allocated objects are freed, and * objcg->nr_charged_bytes can't have an arbitrary byte value. * However, it can be PAGE_SIZE or (x * PAGE_SIZE). * * The following sequence can lead to it: * 1) CPU0: objcg cached in one of stock->cached[i] * 2) CPU1: we do a small allocation (e.g. 92 bytes), * PAGE_SIZE bytes are charged * 3) CPU1: a process from another memcg is allocating something, * the stock if flushed, * objcg->nr_charged_bytes = PAGE_SIZE - 92 * 4) CPU0: we do release this object, * 92 bytes are added to stock->nr_bytes[i] * 5) CPU0: stock is flushed, * 92 bytes are added to objcg->nr_charged_bytes * * In the result, nr_charged_bytes == PAGE_SIZE. * This page will be uncharged in obj_cgroup_release(). */ nr_bytes = atomic_read(&objcg->nr_charged_bytes); WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); nr_pages = nr_bytes >> PAGE_SHIFT;
if (nr_pages) { struct mem_cgroup *memcg;
memcg = get_mem_cgroup_from_objcg(objcg); memcg_uncharge_kmem(memcg, nr_pages); mem_cgroup_put(memcg); }
spin_lock_irqsave(&objcg_lock, flags); list_del(&objcg->list); spin_unlock_irqrestore(&objcg_lock, flags);
percpu_ref_exit(ref); kfree_rcu(objcg, rcu);}
static struct obj_cgroup *obj_cgroup_alloc(void){ struct obj_cgroup *objcg; int ret;
objcg = kzalloc_obj(struct obj_cgroup); if (!objcg) return NULL;
ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, GFP_KERNEL); if (ret) { kfree(objcg); return NULL; } INIT_LIST_HEAD(&objcg->list); return objcg;}
static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){ struct obj_cgroup *objcg, *iter; struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; struct mem_cgroup_per_node *parent_pn = parent->nodeinfo[nid];
objcg = rcu_replace_pointer(pn->objcg, NULL, true); /* 1) Ready to reparent active objcg. */ list_add(&objcg->list, &pn->objcg_list); /* 2) Reparent active objcg and already reparented objcgs to parent. */ list_for_each_entry(iter, &pn->objcg_list, list) WRITE_ONCE(iter->memcg, parent); /* 3) Move already reparented objcgs to the parent's list */ list_splice(&pn->objcg_list, &parent_pn->objcg_list);
return objcg;}
static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force);
/* * Reparent the non-hierarchical lruvec stats that count_shadow_nodes() reads * to approximate the shadow node budget. They are not exposed to userspace * on cgroup v2, but they must follow the reparented folios; otherwise the * ancestor would only receive the negative deltas when the folios are freed * without ever having received the positive base, and its local stats would * permanently underflow. */static void reparent_v2_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent){ int i;
for (i = 0; i < NR_LRU_LISTS; i++) reparent_memcg_lruvec_state_local(memcg, parent, NR_LRU_BASE + i);
reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_RECLAIMABLE_B); reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_UNRECLAIMABLE_B);}
static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent){ /* * Reparent stats exposed non-hierarchically. Flush @memcg's stats first * to read its stats accurately , and conservatively flush @parent's * stats after reparenting to avoid hiding a potentially large stat * update (e.g. from callers of mem_cgroup_flush_stats_ratelimited()). */ __mem_cgroup_flush_stats(memcg, true);
if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { reparent_v2_lruvec_state_local(memcg, parent); } else {#ifdef CONFIG_MEMCG_V1 /* The following counts are all non-hierarchical and need to be reparented. */ reparent_memcg1_state_local(memcg, parent); reparent_memcg1_lruvec_state_local(memcg, parent);#endif }
__mem_cgroup_flush_stats(parent, true);}
static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){ spin_lock_irq(&objcg_lock); spin_lock_nested(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock, 1); spin_lock_nested(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock, 2);}
static inline void reparent_unlocks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){ spin_unlock(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock); spin_unlock(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock); spin_unlock_irq(&objcg_lock);}
static void memcg_reparent_objcgs(struct mem_cgroup *memcg){ struct obj_cgroup *objcg; struct mem_cgroup *parent = parent_mem_cgroup(memcg); int nid;
for_each_node(nid) {retry: if (lru_gen_enabled()) max_lru_gen_memcg(parent, nid);
reparent_locks(memcg, parent, nid);
if (lru_gen_enabled()) { if (!recheck_lru_gen_max_memcg(parent, nid)) { reparent_unlocks(memcg, parent, nid); cond_resched(); goto retry; } lru_gen_reparent_memcg(memcg, parent, nid); } else { lru_reparent_memcg(memcg, parent, nid); }
objcg = __memcg_reparent_objcgs(memcg, parent, nid);
reparent_unlocks(memcg, parent, nid);
percpu_ref_kill(&objcg->refcnt); }
reparent_state_local(memcg, parent);}
/* * A lot of the calls to the cache allocation functions are expected to be * inlined by the compiler. Since the calls to memcg_slab_post_alloc_hook() are * conditional to this static branch, we'll have to allow modules that does * kmem_cache_alloc and the such to see this symbol as well */DEFINE_STATIC_KEY_FALSE(memcg_kmem_online_key);EXPORT_SYMBOL(memcg_kmem_online_key);
DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key);EXPORT_SYMBOL(memcg_bpf_enabled_key);
/** * get_mem_cgroup_css_from_folio - acquire a css of the memcg associated with a folio * @folio: folio of interest * * If memcg is bound to the default hierarchy, css of the memcg associated * with @folio is returned. The returned css remains associated with @folio * until it is released. * * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup * is returned. */struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio){ struct mem_cgroup *memcg;
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return &root_mem_cgroup->css;
memcg = get_mem_cgroup_from_folio(folio);
return memcg ? &memcg->css : &root_mem_cgroup->css;}
/** * page_cgroup_ino - return inode number of the memcg a page is charged to * @page: the page * * Look up the closest online ancestor of the memory cgroup @page is charged to * and return its inode number or 0 if @page is not charged to any cgroup. It * is safe to call this function without holding a reference to @page. * * Note, this function is inherently racy, because there is nothing to prevent * the cgroup inode from getting torn down and potentially reallocated a moment * after page_cgroup_ino() returns, so it only should be used by callers that * do not care (such as procfs interfaces). */ino_t page_cgroup_ino(const struct page *page){ struct mem_cgroup *memcg; unsigned long ino = 0;
rcu_read_lock(); /* page_folio() is racy here, but the entire function is racy anyway */ memcg = folio_memcg_check(page_folio(page));
while (memcg && !css_is_online(&memcg->css)) memcg = parent_mem_cgroup(memcg); if (memcg) ino = cgroup_ino(memcg->css.cgroup); rcu_read_unlock(); return ino;}EXPORT_SYMBOL_GPL(page_cgroup_ino);
/* Subset of node_stat_item for memcg stats */static const unsigned int memcg_node_stat_items[] = { NR_INACTIVE_ANON, NR_ACTIVE_ANON, NR_INACTIVE_FILE, NR_ACTIVE_FILE, NR_UNEVICTABLE, NR_SLAB_RECLAIMABLE_B, NR_SLAB_UNRECLAIMABLE_B, WORKINGSET_REFAULT_ANON, WORKINGSET_REFAULT_FILE, WORKINGSET_ACTIVATE_ANON, WORKINGSET_ACTIVATE_FILE, WORKINGSET_RESTORE_ANON, WORKINGSET_RESTORE_FILE, WORKINGSET_NODERECLAIM, NR_ANON_MAPPED, NR_FILE_MAPPED, NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, NR_SHMEM, NR_SHMEM_THPS, NR_FILE_THPS, NR_ANON_THPS, NR_VMSCAN_WRITE, NR_VMALLOC, NR_KERNEL_STACK_KB, NR_PAGETABLE, NR_SECONDARY_PAGETABLE,#ifdef CONFIG_SWAP NR_SWAPCACHE,#endif#ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS,#endif PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, PGDEMOTE_PROACTIVE, PGSTEAL_KSWAPD, PGSTEAL_DIRECT, PGSTEAL_KHUGEPAGED, PGSTEAL_PROACTIVE, PGSTEAL_ANON, PGSTEAL_FILE, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_KHUGEPAGED, PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, PGROTATE_ANON, PGROTATE_FILE, PGREFILL,#ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB,#endif};
static const unsigned int memcg_stat_items[] = { MEMCG_SWAP, MEMCG_SOCK, MEMCG_PERCPU_B, MEMCG_KMEM, MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP,};
#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items)#define MEMCG_VMSTAT_SIZE (NR_MEMCG_NODE_STAT_ITEMS + \ ARRAY_SIZE(memcg_stat_items))#define BAD_STAT_IDX(index) ((u32)(index) >= U8_MAX)static u8 mem_cgroup_stats_index[MEMCG_NR_STAT] __read_mostly;
static void init_memcg_stats(void){ u8 i, j = 0;
BUILD_BUG_ON(MEMCG_NR_STAT >= U8_MAX);
memset(mem_cgroup_stats_index, U8_MAX, sizeof(mem_cgroup_stats_index));
for (i = 0; i < NR_MEMCG_NODE_STAT_ITEMS; ++i, ++j) mem_cgroup_stats_index[memcg_node_stat_items[i]] = j;
for (i = 0; i < ARRAY_SIZE(memcg_stat_items); ++i, ++j) mem_cgroup_stats_index[memcg_stat_items[i]] = j;}
static inline int memcg_stats_index(int idx){ return mem_cgroup_stats_index[idx];}
struct lruvec_stats_percpu { /* Local (CPU and cgroup) state */ long state[NR_MEMCG_NODE_STAT_ITEMS];
/* Delta calculation for lockless upward propagation */ long state_prev[NR_MEMCG_NODE_STAT_ITEMS];};
struct lruvec_stats { /* Aggregated (CPU and subtree) state */ long state[NR_MEMCG_NODE_STAT_ITEMS];
/* Non-hierarchical (CPU aggregated) state */ long state_local[NR_MEMCG_NODE_STAT_ITEMS];
/* Pending child counts during tree propagation */ long state_pending[NR_MEMCG_NODE_STAT_ITEMS];};
unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx){ const struct mem_cgroup_per_node *pn; long x; int i;
if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx);
i = memcg_stats_index(idx); if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0;
pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); x = READ_ONCE(pn->lruvec_stats->state[i]);#ifdef CONFIG_SMP if (x < 0) x = 0;#endif return x;}
/** * lruvec_page_state_monotonic - non-clamping lruvec stat read for delta sampling * @lruvec: the LRU vector to read from * @idx: the node_stat_item to read * * Returns the raw state[idx] value cast to unsigned long, skipping the * clamp-negative-to-zero step in lruvec_page_state(). Intended for callers * that snapshot a monotonically-incremented counter and subtract two * samples: unsigned modular arithmetic then yields the correct delta across * a signed-long wraparound (a real hazard on 32-bit) that the clamp would * otherwise turn into a huge spurious delta. * * Do NOT use for non-monotonic page-count reads where a transient negative * reading from per-CPU delta skew must present as zero. * * XXX: This helper (and its node/global peers) exists because some * monotonically-incremented event counters are stored in * enum node_stat_item. */unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx){ const struct mem_cgroup_per_node *pn; int i;
if (mem_cgroup_disabled()) return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
i = memcg_stats_index(idx); if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0;
pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]);}
unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx){ const struct mem_cgroup_per_node *pn; long x; int i;
if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx);
i = memcg_stats_index(idx); if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0;
pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); x = READ_ONCE(pn->lruvec_stats->state_local[i]);#ifdef CONFIG_SMP if (x < 0) x = 0;#endif return x;}
static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn, enum node_stat_item idx, long val);
void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent, int idx){ int nid;
for_each_node(nid) { struct lruvec *child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); struct lruvec *parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid)); unsigned long value = lruvec_page_state_local(child_lruvec, idx); struct mem_cgroup_per_node *child_pn, *parent_pn;
child_pn = container_of(child_lruvec, struct mem_cgroup_per_node, lruvec); parent_pn = container_of(parent_lruvec, struct mem_cgroup_per_node, lruvec);
__mod_memcg_lruvec_state(child_pn, idx, -value); __mod_memcg_lruvec_state(parent_pn, idx, value); }}
/* Subset of vm_event_item to report for memcg event stats */static const unsigned int memcg_vm_event_stat[] = {#ifdef CONFIG_MEMCG_V1 PGPGIN, PGPGOUT,#endif PSWPIN, PSWPOUT, PGFAULT, PGMAJFAULT, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE, PGLAZYFREED,#ifdef CONFIG_SWAP SWPIN_ZERO, SWPOUT_ZERO,#endif#ifdef CONFIG_ZSWAP ZSWPIN, ZSWPOUT, ZSWPWB,#endif#ifdef CONFIG_TRANSPARENT_HUGEPAGE THP_FAULT_ALLOC, THP_COLLAPSE_ALLOC, THP_SWPOUT, THP_SWPOUT_FALLBACK,#endif#ifdef CONFIG_NUMA_BALANCING NUMA_PAGE_MIGRATE, NUMA_PTE_UPDATES, NUMA_HINT_FAULTS,#endif};
#define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat)static u8 mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly;
static void init_memcg_events(void){ u8 i;
BUILD_BUG_ON(NR_VM_EVENT_ITEMS >= U8_MAX);
memset(mem_cgroup_events_index, U8_MAX, sizeof(mem_cgroup_events_index));
for (i = 0; i < NR_MEMCG_EVENTS; ++i) mem_cgroup_events_index[memcg_vm_event_stat[i]] = i;}
static inline int memcg_events_index(enum vm_event_item idx){ return mem_cgroup_events_index[idx];}
struct memcg_vmstats_percpu { /* Stats updates since the last flush */ unsigned long stats_updates;
/* Cached pointers for fast iteration in memcg_rstat_updated() */ struct memcg_vmstats_percpu __percpu *parent_pcpu; struct memcg_vmstats *vmstats;
/* The above should fit a single cacheline for memcg_rstat_updated() */
/* Local (CPU and cgroup) page state & events */ long state[MEMCG_VMSTAT_SIZE]; unsigned long events[NR_MEMCG_EVENTS];
/* Delta calculation for lockless upward propagation */ long state_prev[MEMCG_VMSTAT_SIZE]; unsigned long events_prev[NR_MEMCG_EVENTS];} ____cacheline_aligned;
struct memcg_vmstats { /* Aggregated (CPU and subtree) page state & events */ long state[MEMCG_VMSTAT_SIZE]; unsigned long events[NR_MEMCG_EVENTS];
#ifdef CONFIG_MEMCG_V1 /* Non-hierarchical (CPU aggregated) page state & events */ long state_local[MEMCG_VMSTAT_SIZE]; unsigned long events_local[NR_MEMCG_EVENTS];#endif
/* Pending child counts during tree propagation */ long state_pending[MEMCG_VMSTAT_SIZE]; unsigned long events_pending[NR_MEMCG_EVENTS];
/* Stats updates since the last flush */ atomic_long_t stats_updates;};
/* * The non-hierarchical memcg-wide counters are read back only by the legacy * memory.stat and memory.numa_stat, and by reparenting on offline, all of which * are v1-only. If the kernel is built without CONFIG_MEMCG_V1, or if the boot * param cgroup_no_v1= has blocked the memory controller from v1 hierarchies, * then nothing reads them and writers can skip the updates. */static long *memcg_state_local_array(struct mem_cgroup *memcg){#ifdef CONFIG_MEMCG_V1 if (!cgroup1_ssid_disabled(memory_cgrp_id)) return memcg->vmstats->state_local;#endif return NULL;}
static unsigned long *memcg_events_local_array(struct mem_cgroup *memcg){#ifdef CONFIG_MEMCG_V1 if (!cgroup1_ssid_disabled(memory_cgrp_id)) return memcg->vmstats->events_local;#endif return NULL;}
/* * memcg and lruvec stats flushing * * Many codepaths leading to stats update or read are performance sensitive and * adding stats flushing in such codepaths is not desirable. So, to optimize the * flushing the kernel does: * * 1) Periodically and asynchronously flush the stats every 2 seconds to not let * rstat update tree grow unbounded. * * 2) Flush the stats synchronously on reader side only when there are more than * (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization * will let stats be out of sync by at most (MEMCG_CHARGE_BATCH * nr_cpus) but * only for 2 seconds due to (1). */static void flush_memcg_stats_dwork(struct work_struct *w);static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);static u64 flush_last_time;
#define FLUSH_TIME (2UL*HZ)
static bool memcg_vmstats_needs_flush(struct memcg_vmstats *vmstats){ return atomic_long_read(&vmstats->stats_updates) > MEMCG_CHARGE_BATCH * num_online_cpus();}
static inline void memcg_rstat_updated(struct mem_cgroup *memcg, long val, int cpu){ struct memcg_vmstats_percpu __percpu *statc_pcpu; struct memcg_vmstats_percpu *statc; unsigned long stats_updates;
if (!val) return;
__css_rstat_updated(&memcg->css, cpu); statc_pcpu = memcg->vmstats_percpu; for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) { statc = this_cpu_ptr(statc_pcpu); /* * If @memcg is already flushable then all its ancestors are * flushable as well and also there is no need to increase * stats_updates. */ if (memcg_vmstats_needs_flush(statc->vmstats)) break;
stats_updates = this_cpu_add_return(statc_pcpu->stats_updates, abs(val)); if (stats_updates < MEMCG_CHARGE_BATCH) continue;
stats_updates = this_cpu_xchg(statc_pcpu->stats_updates, 0); atomic_long_add(stats_updates, &statc->vmstats->stats_updates); }}
static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force){ bool needs_flush = memcg_vmstats_needs_flush(memcg->vmstats);
trace_memcg_flush_stats(memcg, atomic_long_read(&memcg->vmstats->stats_updates), force, needs_flush);
if (!force && !needs_flush) return;
if (mem_cgroup_is_root(memcg)) WRITE_ONCE(flush_last_time, get_jiffies_64());
css_rstat_flush(&memcg->css);}
/* * mem_cgroup_flush_stats - flush the stats of a memory cgroup subtree * @memcg: root of the subtree to flush * * Flushing is serialized by the underlying global rstat lock. There is also a * minimum amount of work to be done even if there are no stat updates to flush. * Hence, we only flush the stats if the updates delta exceeds a threshold. This * avoids unnecessary work and contention on the underlying lock. */void mem_cgroup_flush_stats(struct mem_cgroup *memcg){ if (mem_cgroup_disabled()) return;
if (!memcg) memcg = root_mem_cgroup;
__mem_cgroup_flush_stats(memcg, false);}
void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg){ /* Only flush if the periodic flusher is one full cycle late */ if (time_after64(get_jiffies_64(), READ_ONCE(flush_last_time) + 2 * FLUSH_TIME)) mem_cgroup_flush_stats(memcg);}
static void flush_memcg_stats_dwork(struct work_struct *w){ /* * Deliberately ignore memcg_vmstats_needs_flush() here so that flushing * in latency-sensitive paths is as cheap as possible. */ __mem_cgroup_flush_stats(root_mem_cgroup, true); queue_delayed_work(system_dfl_wq, &stats_flush_dwork, FLUSH_TIME);}
unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx){ long x; int i = memcg_stats_index(idx);
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0;
x = READ_ONCE(memcg->vmstats->state[i]);#ifdef CONFIG_SMP if (x < 0) x = 0;#endif return x;}
bool memcg_stat_item_valid(int idx){ if ((u32)idx >= MEMCG_NR_STAT) return false;
return !BAD_STAT_IDX(memcg_stats_index(idx));}
static int memcg_page_state_unit(int item);
/* * Normalize the value passed into memcg_rstat_updated() to be in pages. Round * up non-zero sub-page updates to 1 page as zero page updates are ignored. */static long memcg_state_val_in_pages(int idx, long val){ int unit = memcg_page_state_unit(idx); long res;
if (!val || unit == PAGE_SIZE) return val;
/* Get the absolute value of (val * unit / PAGE_SIZE). */ res = mult_frac(abs(val), unit, PAGE_SIZE); /* Round up zero values. */ res = res ? : 1;
return val < 0 ? -res : res;}
/* * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race * with reparenting of non-hierarchical state_locals. Offlining a * memcg is rare, so do the redirection for all cgroup hierarchies. */static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg, bool *rcu_locked){ /* * Fast path: the caller holds a reference to @memcg, so reading * its CSS_DYING flag without the RCU lock is safe. The RCU lock * is only needed to walk up to a non-dying ancestor, which * happens only while a memcg is actually being offlined. */ if (!memcg_is_dying(memcg)) { *rcu_locked = false; return memcg; }
rcu_read_lock(); *rcu_locked = true;
while (memcg_is_dying(memcg)) memcg = parent_mem_cgroup(memcg);
return memcg;}
static inline void get_non_dying_memcg_end(bool rcu_locked){ if (!rcu_locked) return;
rcu_read_unlock();}
static void __mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, long val){ int i = memcg_stats_index(idx); int cpu;
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return;
cpu = get_cpu();
this_cpu_add(memcg->vmstats_percpu->state[i], val); val = memcg_state_val_in_pages(idx, val); memcg_rstat_updated(memcg, val, cpu);
trace_mod_memcg_state(memcg, idx, val);
put_cpu();}
/** * mod_memcg_state - update cgroup memory statistics * @memcg: the memory cgroup * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item * @val: delta to add to the counter, can be negative */void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, int val){ bool rcu_locked = false;
if (mem_cgroup_disabled()) return;
memcg = get_non_dying_memcg_start(memcg, &rcu_locked); __mod_memcg_state(memcg, idx, val); get_non_dying_memcg_end(rcu_locked);}
#ifdef CONFIG_MEMCG_V1/* idx can be of type enum memcg_stat_item or node_stat_item. */unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx){ long x; int i = memcg_stats_index(idx);
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return 0;
x = READ_ONCE(memcg->vmstats->state_local[i]);#ifdef CONFIG_SMP if (x < 0) x = 0;#endif return x;}
void reparent_memcg_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent, int idx){ unsigned long value = memcg_page_state_local(memcg, idx);
__mod_memcg_state(memcg, idx, -value); __mod_memcg_state(parent, idx, value);}#endif
static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn, enum node_stat_item idx, long val){ struct mem_cgroup *memcg = pn->memcg; int i = memcg_stats_index(idx); int cpu;
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return;
cpu = get_cpu();
/* Update memcg */ this_cpu_add(memcg->vmstats_percpu->state[i], val);
/* Update lruvec */ this_cpu_add(pn->lruvec_stats_percpu->state[i], val);
val = memcg_state_val_in_pages(idx, val); memcg_rstat_updated(memcg, val, cpu); trace_mod_memcg_lruvec_state(memcg, idx, val);
put_cpu();}
void mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val){ struct pglist_data *pgdat = lruvec_pgdat(lruvec); struct mem_cgroup_per_node *pn; struct mem_cgroup *memcg; bool rcu_locked = false;
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); memcg = get_non_dying_memcg_start(pn->memcg, &rcu_locked); pn = memcg->nodeinfo[pgdat->node_id];
__mod_memcg_lruvec_state(pn, idx, val);
get_non_dying_memcg_end(rcu_locked);}
/** * mod_lruvec_state - update lruvec memory statistics * @lruvec: the lruvec * @idx: the stat item * @val: delta to add to the counter, can be negative * * The lruvec is the intersection of the NUMA node and a cgroup. This * function updates the all three counters that are affected by a * change of state at this level: per-node, per-cgroup, per-lruvec. */void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val){ /* Update node */ mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
/* Update memcg and lruvec */ if (!mem_cgroup_disabled()) mod_memcg_lruvec_state(lruvec, idx, val);}
void lruvec_stat_mod_folio(struct folio *folio, enum node_stat_item idx, int val){ struct mem_cgroup *memcg; pg_data_t *pgdat = folio_pgdat(folio); struct lruvec *lruvec;
rcu_read_lock(); memcg = folio_memcg(folio); /* Untracked pages have no memcg, no lruvec. Update only the node */ if (!memcg) { rcu_read_unlock(); mod_node_page_state(pgdat, idx, val); return; }
lruvec = mem_cgroup_lruvec(memcg, pgdat); mod_lruvec_state(lruvec, idx, val); rcu_read_unlock();}EXPORT_SYMBOL(lruvec_stat_mod_folio);
void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val){ pg_data_t *pgdat = page_pgdat(virt_to_page(p)); struct mem_cgroup *memcg; struct lruvec *lruvec;
rcu_read_lock(); memcg = mem_cgroup_from_virt(p);
/* * Untracked pages have no memcg, no lruvec. Update only the * node. If we reparent the slab objects to the root memcg, * when we free the slab object, we need to update the per-memcg * vmstats to keep it correct for the root memcg. */ if (!memcg) { mod_node_page_state(pgdat, idx, val); } else { lruvec = mem_cgroup_lruvec(memcg, pgdat); mod_lruvec_state(lruvec, idx, val); } rcu_read_unlock();}
/** * count_memcg_events - account VM events in a cgroup * @memcg: the memory cgroup * @idx: the event item * @count: the number of events that occurred */void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count){ int i = memcg_events_index(idx); int cpu;
if (mem_cgroup_disabled()) return;
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx)) return;
cpu = get_cpu();
this_cpu_add(memcg->vmstats_percpu->events[i], count); memcg_rstat_updated(memcg, count, cpu); trace_count_memcg_events(memcg, idx, count);
put_cpu();}
unsigned long memcg_events(const struct mem_cgroup *memcg, int event){ int i = memcg_events_index(event);
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, event)) return 0;
return READ_ONCE(memcg->vmstats->events[i]);}
bool memcg_vm_event_item_valid(enum vm_event_item idx){ if (idx >= NR_VM_EVENT_ITEMS) return false;
return !BAD_STAT_IDX(memcg_events_index(idx));}
#ifdef CONFIG_MEMCG_V1unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event){ int i = memcg_events_index(event);
if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, event)) return 0;
return READ_ONCE(memcg->vmstats->events_local[i]);}#endif
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p){ /* * mm_update_next_owner() may clear mm->owner to NULL * if it races with swapoff, page migration, etc. * So this can be called with p == NULL. */ if (unlikely(!p)) return NULL;
return mem_cgroup_from_css(task_css(p, memory_cgrp_id));}EXPORT_SYMBOL(mem_cgroup_from_task);
static __always_inline struct mem_cgroup *active_memcg(void){ if (!in_task()) return this_cpu_read(int_active_memcg); else return current->active_memcg;}
/** * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg. * @mm: mm from which memcg should be extracted. It can be NULL. * * Obtain a reference on mm->memcg and returns it if successful. If mm * is NULL, then the memcg is chosen as follows: * 1) The active memcg, if set. * 2) current->mm->memcg, if available * 3) root memcg * If mem_cgroup is disabled, NULL is returned. */struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return NULL;
/* * Page cache insertions can happen without an * actual mm context, e.g. during disk probing * on boot, loopback IO, acct() writes etc. * * No need to css_get on root memcg as the reference * counting is disabled on the root level in the * cgroup core. See CSS_NO_REF. */ if (unlikely(!mm)) { memcg = active_memcg(); if (unlikely(memcg)) { /* remote memcg must hold a ref */ css_get(&memcg->css); return memcg; } mm = current->mm; if (unlikely(!mm)) return root_mem_cgroup; }
rcu_read_lock(); do { memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (unlikely(!memcg)) memcg = root_mem_cgroup; } while (!css_tryget(&memcg->css)); rcu_read_unlock(); return memcg;}EXPORT_SYMBOL(get_mem_cgroup_from_mm);
/** * get_mem_cgroup_from_current - Obtain a reference on current task's memcg. */struct mem_cgroup *get_mem_cgroup_from_current(void){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return NULL;
again: rcu_read_lock(); memcg = mem_cgroup_from_task(current); if (!css_tryget(&memcg->css)) { rcu_read_unlock(); goto again; } rcu_read_unlock(); return memcg;}
/** * get_mem_cgroup_from_folio - Obtain a reference on a given folio's memcg. * @folio: folio from which memcg should be extracted. * * See folio_memcg() for folio->objcg/memcg binding rules. */struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio){ struct mem_cgroup *memcg;
if (mem_cgroup_disabled()) return NULL;
if (!folio_memcg_charged(folio)) return root_mem_cgroup;
rcu_read_lock(); do { memcg = folio_memcg(folio); } while (unlikely(!css_tryget(&memcg->css))); rcu_read_unlock(); return memcg;}
/** * mem_cgroup_iter - iterate over memory cgroup hierarchy * @root: hierarchy root * @prev: previously returned memcg, NULL on first invocation * @reclaim: cookie for shared reclaim walks, NULL for full walks * * Returns references to children of the hierarchy below @root, or * @root itself, or %NULL after a full round-trip. * * Caller must pass the return value in @prev on subsequent * invocations for reference counting, or use mem_cgroup_iter_break() * to cancel a hierarchy walk before the round-trip is complete. * * Reclaimers can specify a node in @reclaim to divide up the memcgs * in the hierarchy among all concurrent reclaimers operating on the * same node. */struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, struct mem_cgroup *prev, struct mem_cgroup_reclaim_cookie *reclaim){ struct mem_cgroup_reclaim_iter *iter; struct cgroup_subsys_state *css; struct mem_cgroup *pos; struct mem_cgroup *next;
if (mem_cgroup_disabled()) return NULL;
if (!root) root = root_mem_cgroup;
rcu_read_lock();restart: next = NULL;
if (reclaim) { int gen; int nid = reclaim->pgdat->node_id;
iter = &root->nodeinfo[nid]->iter; gen = atomic_read(&iter->generation);
/* * On start, join the current reclaim iteration cycle. * Exit when a concurrent walker completes it. */ if (!prev) reclaim->generation = gen; else if (reclaim->generation != gen) goto out_unlock;
pos = READ_ONCE(iter->position); } else pos = prev;
css = pos ? &pos->css : NULL;
while ((css = css_next_descendant_pre(css, &root->css))) { /* * Verify the css and acquire a reference. The root * is provided by the caller, so we know it's alive * and kicking, and don't take an extra reference. */ if (css == &root->css || css_tryget(css)) break; }
next = mem_cgroup_from_css(css);
if (reclaim) { /* * The position could have already been updated by a competing * thread, so check that the value hasn't changed since we read * it to avoid reclaiming from the same cgroup twice. */ if (cmpxchg(&iter->position, pos, next) != pos) { if (css && css != &root->css) css_put(css); goto restart; }
if (!next) { atomic_inc(&iter->generation);
/* * Reclaimers share the hierarchy walk, and a * new one might jump in right at the end of * the hierarchy - make sure they see at least * one group and restart from the beginning. */ if (!prev) goto restart; } }
out_unlock: rcu_read_unlock(); if (prev && prev != root) css_put(&prev->css);
return next;}
/** * mem_cgroup_iter_break - abort a hierarchy walk prematurely * @root: hierarchy root * @prev: last visited hierarchy member as returned by mem_cgroup_iter() */void mem_cgroup_iter_break(struct mem_cgroup *root, struct mem_cgroup *prev){ if (!root) root = root_mem_cgroup; if (prev && prev != root) css_put(&prev->css);}
static void __invalidate_reclaim_iterators(struct mem_cgroup *from, struct mem_cgroup *dead_memcg){ struct mem_cgroup_reclaim_iter *iter; struct mem_cgroup_per_node *mz; int nid;
for_each_node(nid) { mz = from->nodeinfo[nid]; iter = &mz->iter; cmpxchg(&iter->position, dead_memcg, NULL); }}
static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg){ struct mem_cgroup *memcg = dead_memcg; struct mem_cgroup *last;
do { __invalidate_reclaim_iterators(memcg, dead_memcg); last = memcg; } while ((memcg = parent_mem_cgroup(memcg)));
/* * When cgroup1 non-hierarchy mode is used, * parent_mem_cgroup() does not walk all the way up to the * cgroup root (root_mem_cgroup). So we have to handle * dead_memcg from cgroup root separately. */ if (!mem_cgroup_is_root(last)) __invalidate_reclaim_iterators(root_mem_cgroup, dead_memcg);}
/** * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy * @memcg: hierarchy root * @fn: function to call for each task * @arg: argument passed to @fn * * This function iterates over tasks attached to @memcg or to any of its * descendants and calls @fn for each task. If @fn returns a non-zero * value, the function breaks the iteration loop. Otherwise, it will iterate * over all tasks and return 0. * * This function must not be called for the root memory cgroup. */void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*fn)(struct task_struct *, void *), void *arg){ struct mem_cgroup *iter; int ret = 0;
BUG_ON(mem_cgroup_is_root(memcg));
for_each_mem_cgroup_tree(iter, memcg) { struct css_task_iter it; struct task_struct *task;
css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it); while (!ret && (task = css_task_iter_next(&it))) { ret = fn(task, arg); /* Avoid potential softlockup warning */ cond_resched(); } css_task_iter_end(&it); if (ret) { mem_cgroup_iter_break(memcg, iter); break; } }}
/** * folio_lruvec_lock - Lock the lruvec for a folio. * @folio: Pointer to the folio. * * These functions are safe to use under any of the following conditions: * - folio locked * - folio_test_lru false * - folio frozen (refcount of 0) * * Return: The lruvec this folio is on with its lock held and rcu read lock held. */struct lruvec *folio_lruvec_lock(const struct folio *folio){ struct lruvec *lruvec;
rcu_read_lock();retry: lruvec = folio_lruvec(folio); spin_lock(&lruvec->lru_lock); if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) { spin_unlock(&lruvec->lru_lock); goto retry; }
return lruvec;}
/** * folio_lruvec_lock_irq - Lock the lruvec for a folio. * @folio: Pointer to the folio. * * These functions are safe to use under any of the following conditions: * - folio locked * - folio_test_lru false * - folio frozen (refcount of 0) * * Return: The lruvec this folio is on with its lock held and interrupts * disabled and rcu read lock held. */struct lruvec *folio_lruvec_lock_irq(const struct folio *folio){ struct lruvec *lruvec;
rcu_read_lock();retry: lruvec = folio_lruvec(folio); spin_lock_irq(&lruvec->lru_lock); if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) { spin_unlock_irq(&lruvec->lru_lock); goto retry; }
return lruvec;}
/** * folio_lruvec_lock_irqsave - Lock the lruvec for a folio. * @folio: Pointer to the folio. * @flags: Pointer to irqsave flags. * * These functions are safe to use under any of the following conditions: * - folio locked * - folio_test_lru false * - folio frozen (refcount of 0) * * Return: The lruvec this folio is on with its lock held and interrupts * disabled and rcu read lock held. */struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags){ struct lruvec *lruvec;
rcu_read_lock();retry: lruvec = folio_lruvec(folio); spin_lock_irqsave(&lruvec->lru_lock, *flags); if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) { spin_unlock_irqrestore(&lruvec->lru_lock, *flags); goto retry; }
return lruvec;}
/** * mem_cgroup_update_lru_size - account for adding or removing an lru page * @lruvec: mem_cgroup per zone lru vector * @lru: index of lru list the page is sitting on * @zid: zone id of the accounted pages * @nr_pages: positive when adding or negative when removing * * This function must be called under lru_lock, just before a page is added * to or just after a page is removed from an lru list. */void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages){ struct mem_cgroup_per_node *mz;
if (mem_cgroup_disabled()) return;
mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); mz->lru_zone_size[zid][lru] += nr_pages;}
/** * mem_cgroup_margin - calculate chargeable space of a memory cgroup * @memcg: the memory cgroup * * Returns the maximum amount of memory @mem can be charged with, in * pages. */static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg){ unsigned long margin = 0; unsigned long count; unsigned long limit;
count = page_counter_read(&memcg->memory); limit = READ_ONCE(memcg->memory.max); if (count < limit) margin = limit - count;
if (do_memsw_account()) { count = page_counter_read(&memcg->memsw); limit = READ_ONCE(memcg->memsw.max); if (count < limit) margin = min(margin, limit - count); else margin = 0; }
return margin;}
struct memory_stat { const char *name; unsigned int idx;};
static const struct memory_stat memory_stats[] = { { "anon", NR_ANON_MAPPED }, { "file", NR_FILE_PAGES }, { "kernel", MEMCG_KMEM }, { "kernel_stack", NR_KERNEL_STACK_KB }, { "pagetables", NR_PAGETABLE }, { "sec_pagetables", NR_SECONDARY_PAGETABLE }, { "percpu", MEMCG_PERCPU_B }, { "sock", MEMCG_SOCK }, { "vmalloc", NR_VMALLOC }, { "shmem", NR_SHMEM },#ifdef CONFIG_ZSWAP { "zswap", MEMCG_ZSWAP_B }, { "zswapped", MEMCG_ZSWAPPED }, { "zswap_incomp", MEMCG_ZSWAP_INCOMP },#endif { "file_mapped", NR_FILE_MAPPED }, { "file_dirty", NR_FILE_DIRTY }, { "file_writeback", NR_WRITEBACK },#ifdef CONFIG_SWAP { "swapcached", NR_SWAPCACHE },#endif#ifdef CONFIG_TRANSPARENT_HUGEPAGE { "anon_thp", NR_ANON_THPS }, { "file_thp", NR_FILE_THPS }, { "shmem_thp", NR_SHMEM_THPS },#endif { "inactive_anon", NR_INACTIVE_ANON }, { "active_anon", NR_ACTIVE_ANON }, { "inactive_file", NR_INACTIVE_FILE }, { "active_file", NR_ACTIVE_FILE }, { "unevictable", NR_UNEVICTABLE }, { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B }, { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B },#ifdef CONFIG_HUGETLB_PAGE { "hugetlb", NR_HUGETLB },#endif
/* The memory events */ { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, { "workingset_refault_file", WORKINGSET_REFAULT_FILE }, { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON }, { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE }, { "workingset_restore_anon", WORKINGSET_RESTORE_ANON }, { "workingset_restore_file", WORKINGSET_RESTORE_FILE }, { "workingset_nodereclaim", WORKINGSET_NODERECLAIM },
{ "pgdemote_kswapd", PGDEMOTE_KSWAPD }, { "pgdemote_direct", PGDEMOTE_DIRECT }, { "pgdemote_khugepaged", PGDEMOTE_KHUGEPAGED }, { "pgdemote_proactive", PGDEMOTE_PROACTIVE }, { "pgsteal_kswapd", PGSTEAL_KSWAPD }, { "pgsteal_direct", PGSTEAL_DIRECT }, { "pgsteal_khugepaged", PGSTEAL_KHUGEPAGED }, { "pgsteal_proactive", PGSTEAL_PROACTIVE }, { "pgscan_kswapd", PGSCAN_KSWAPD }, { "pgscan_direct", PGSCAN_DIRECT }, { "pgscan_khugepaged", PGSCAN_KHUGEPAGED }, { "pgscan_proactive", PGSCAN_PROACTIVE }, { "pgrefill", PGREFILL },#ifdef CONFIG_NUMA_BALANCING { "pgpromote_success", PGPROMOTE_SUCCESS },#endif};
/* The actual unit of the state item, not the same as the output unit */static int memcg_page_state_unit(int item){ switch (item) { case MEMCG_PERCPU_B: case MEMCG_ZSWAP_B: case NR_SLAB_RECLAIMABLE_B: case NR_SLAB_UNRECLAIMABLE_B: return 1; case NR_KERNEL_STACK_KB: return SZ_1K; default: return PAGE_SIZE; }}
/* Translate stat items to the correct unit for memory.stat output */static int memcg_page_state_output_unit(int item){ /* * Workingset state is actually in pages, but we export it to userspace * as a scalar count of events, so special case it here. * * Demotion and promotion activities are exported in pages, consistent * with their global counterparts. */ switch (item) { case WORKINGSET_REFAULT_ANON: case WORKINGSET_REFAULT_FILE: case WORKINGSET_ACTIVATE_ANON: case WORKINGSET_ACTIVATE_FILE: case WORKINGSET_RESTORE_ANON: case WORKINGSET_RESTORE_FILE: case WORKINGSET_NODERECLAIM: case PGDEMOTE_KSWAPD: case PGDEMOTE_DIRECT: case PGDEMOTE_KHUGEPAGED: case PGDEMOTE_PROACTIVE: case PGSTEAL_KSWAPD: case PGSTEAL_DIRECT: case PGSTEAL_KHUGEPAGED: case PGSTEAL_PROACTIVE: case PGSCAN_KSWAPD: case PGSCAN_DIRECT: case PGSCAN_KHUGEPAGED: case PGSCAN_PROACTIVE: case PGREFILL:#ifdef CONFIG_NUMA_BALANCING case PGPROMOTE_SUCCESS:#endif return 1; default: return memcg_page_state_unit(item); }}
unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item){ return memcg_page_state(memcg, item) * memcg_page_state_output_unit(item);}
#ifdef CONFIG_MEMCG_V1unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item){ return memcg_page_state_local(memcg, item) * memcg_page_state_output_unit(item);}#endif
#ifdef CONFIG_HUGETLB_PAGEstatic bool memcg_accounts_hugetlb(void){ return cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_HUGETLB_ACCOUNTING;}#else /* CONFIG_HUGETLB_PAGE */static bool memcg_accounts_hugetlb(void){ return false;}#endif /* CONFIG_HUGETLB_PAGE */
static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s){ int i;
/* * Provide statistics on the state of the memory subsystem as * well as cumulative event counters that show past behavior. * * This list is ordered following a combination of these gradients: * 1) generic big picture -> specifics and details * 2) reflecting userspace activity -> reflecting kernel heuristics * * Current memory state: */ mem_cgroup_flush_stats(memcg);
for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { u64 size;
#ifdef CONFIG_HUGETLB_PAGE if (unlikely(memory_stats[i].idx == NR_HUGETLB) && !memcg_accounts_hugetlb()) continue;#endif size = memcg_page_state_output(memcg, memory_stats[i].idx); seq_buf_printf(s, "%s %llu\n", memory_stats[i].name, size);
if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) { size += memcg_page_state_output(memcg, NR_SLAB_RECLAIMABLE_B); seq_buf_printf(s, "slab %llu\n", size); } }
/* Accumulated memory events */ seq_buf_printf(s, "pgscan %lu\n", memcg_page_state(memcg, PGSCAN_KSWAPD) + memcg_page_state(memcg, PGSCAN_DIRECT) + memcg_page_state(memcg, PGSCAN_PROACTIVE) + memcg_page_state(memcg, PGSCAN_KHUGEPAGED)); seq_buf_printf(s, "pgsteal %lu\n", memcg_page_state(memcg, PGSTEAL_KSWAPD) + memcg_page_state(memcg, PGSTEAL_DIRECT) + memcg_page_state(memcg, PGSTEAL_PROACTIVE) + memcg_page_state(memcg, PGSTEAL_KHUGEPAGED));
for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {#ifdef CONFIG_MEMCG_V1 if (memcg_vm_event_stat[i] == PGPGIN || memcg_vm_event_stat[i] == PGPGOUT) continue;#endif seq_buf_printf(s, "%s %lu\n", vm_event_name(memcg_vm_event_stat[i]), memcg_events(memcg, memcg_vm_event_stat[i])); }}
static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s){ if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) memcg_stat_format(memcg, s); else memcg1_stat_format(memcg, s); if (seq_buf_has_overflowed(s)) pr_warn("%s: Warning, stat buffer overflow, please report\n", __func__);}
/** * mem_cgroup_print_oom_context: Print OOM information relevant to * memory controller. * @memcg: The memory cgroup that went over limit * @p: Task that is going to be killed * * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is * enabled */void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p){ rcu_read_lock();
if (memcg) { pr_cont(",oom_memcg="); pr_cont_cgroup_path(memcg->css.cgroup); } else pr_cont(",global_oom"); if (p) { pr_cont(",task_memcg="); pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id)); } rcu_read_unlock();}
/** * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to * memory controller. * @memcg: The memory cgroup that went over limit */void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg){ /* Use static buffer, for the caller is holding oom_lock. */ static char buf[SEQ_BUF_SIZE]; struct seq_buf s; unsigned long memory_failcnt;
lockdep_assert_held(&oom_lock);
if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) memory_failcnt = atomic_long_read(&memcg->memory_events[MEMCG_MAX]); else memory_failcnt = memcg->memory.failcnt;
pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->memory)), K((u64)READ_ONCE(memcg->memory.max)), memory_failcnt); if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->swap)), K((u64)READ_ONCE(memcg->swap.max)), atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));#ifdef CONFIG_MEMCG_V1 else { pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->memsw)), K((u64)memcg->memsw.max), memcg->memsw.failcnt); pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n", K((u64)page_counter_read(&memcg->kmem)), K((u64)memcg->kmem.max), memcg->kmem.failcnt); }#endif
pr_info("Memory cgroup stats for "); pr_cont_cgroup_path(memcg->css.cgroup); pr_cont(":"); seq_buf_init(&s, buf, SEQ_BUF_SIZE); memory_stat_format(memcg, &s); seq_buf_do_printk(&s, KERN_INFO);}
/* * Return the memory (and swap, if configured) limit for a memcg. */unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg){ unsigned long max = READ_ONCE(memcg->memory.max);
if (do_memsw_account()) { if (mem_cgroup_swappiness(memcg)) { /* Calculate swap excess capacity from memsw limit */ unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
max += min(swap, (unsigned long)total_swap_pages); } } else { if (mem_cgroup_swappiness(memcg)) max += min(READ_ONCE(memcg->swap.max), (unsigned long)total_swap_pages); } return max;}
void __memcg_memory_event(struct mem_cgroup *memcg, enum memcg_memory_event event, bool allow_spinning){ bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX || event == MEMCG_SWAP_FAIL;
/* For now only MEMCG_MAX can happen with !allow_spinning context. */ VM_WARN_ON_ONCE(!allow_spinning && event != MEMCG_MAX);
atomic_long_inc(&memcg->memory_events_local[event]); if (!swap_event && allow_spinning) cgroup_file_notify(&memcg->events_local_file);
do { atomic_long_inc(&memcg->memory_events[event]); if (allow_spinning) { if (swap_event) cgroup_file_notify(&memcg->swap_events_file); else cgroup_file_notify(&memcg->events_file); }
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) break; if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS) break; } while ((memcg = parent_mem_cgroup(memcg)) && !mem_cgroup_is_root(memcg));}EXPORT_SYMBOL_GPL(__memcg_memory_event);
static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, int order){ struct oom_control oc = { .zonelist = NULL, .nodemask = NULL, .memcg = memcg, .gfp_mask = gfp_mask, .order = order, }; bool ret = true;
if (mutex_lock_killable(&oom_lock)) return true;
if (mem_cgroup_margin(memcg) >= (1 << order)) goto unlock;
/* * A few threads which were not waiting at mutex_lock_killable() can * fail to bail out. Therefore, check again after holding oom_lock. */ ret = out_of_memory(&oc);
unlock: mutex_unlock(&oom_lock); return ret;}
/* * Returns true if successfully killed one or more processes. Though in some * corner cases it can return true even without killing any process. */static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order){ bool locked, ret;
if (order > PAGE_ALLOC_COSTLY_ORDER) return false;
memcg_memory_event(memcg, MEMCG_OOM);
if (!memcg1_oom_prepare(memcg, &locked)) return false;
ret = mem_cgroup_out_of_memory(memcg, mask, order);
memcg1_oom_finish(memcg, locked);
return ret;}
/** * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM * @victim: task to be killed by the OOM killer * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM * * Returns a pointer to a memory cgroup, which has to be cleaned up * by killing all belonging OOM-killable tasks. * * Caller has to call mem_cgroup_put() on the returned non-NULL memcg. */struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim, struct mem_cgroup *oom_domain){ struct mem_cgroup *oom_group = NULL; struct mem_cgroup *memcg;
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return NULL;
if (!oom_domain) oom_domain = root_mem_cgroup;
rcu_read_lock();
memcg = mem_cgroup_from_task(victim); if (mem_cgroup_is_root(memcg)) goto out;
/* * If the victim task has been asynchronously moved to a different * memory cgroup, we might end up killing tasks outside oom_domain. * In this case it's better to ignore memory.group.oom. */ if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain))) goto out;
/* * Traverse the memory cgroup hierarchy from the victim task's * cgroup up to the OOMing cgroup (or root) to find the * highest-level memory cgroup with oom.group set. */ for (; memcg; memcg = parent_mem_cgroup(memcg)) { if (READ_ONCE(memcg->oom_group)) oom_group = memcg;
if (memcg == oom_domain) break; }
if (oom_group) css_get(&oom_group->css);out: rcu_read_unlock();
return oom_group;}
void mem_cgroup_print_oom_group(struct mem_cgroup *memcg){ pr_info("Tasks in "); pr_cont_cgroup_path(memcg->css.cgroup); pr_cont(" are going to be killed due to memory.oom.group set\n");}
/* * The value of NR_MEMCG_STOCK is selected to keep the cached memcgs and their * nr_pages in a single cacheline. This may change in future. */#define NR_MEMCG_STOCK 7
/* * Watermarks for a charge stock slot, in the spirit of pcp->high and * pcp->batch: MEMCG_STOCK_HIGH is the high watermark at which a slot is * trimmed, and it is trimmed down to MEMCG_STOCK_LOW rather than emptied. */#define MEMCG_STOCK_LOW (MEMCG_CHARGE_BATCH / 2)#define MEMCG_STOCK_HIGH (MEMCG_CHARGE_BATCH)
#define FLUSHING_CACHED_CHARGE 0struct memcg_stock_pcp { local_trylock_t lock; uint8_t nr_pages[NR_MEMCG_STOCK]; struct mem_cgroup *cached[NR_MEMCG_STOCK];
struct work_struct work; unsigned long flags; uint8_t drain_idx;};
static DEFINE_PER_CPU_ALIGNED(struct memcg_stock_pcp, memcg_stock) = { .lock = INIT_LOCAL_TRYLOCK(lock),};
/* * NR_OBJ_STOCK is sized so the entire hot path of obj_stock_pcp * (lock, accounting metadata, nr_bytes[] and cached[]) fits within a * single 64-byte cache line on non-debug 64-bit builds. With 5 slots: * lock(1) + index(1) + node_id(2) + slab stats(4) + nr_bytes(10) * + pad(6) + cached(40) == 64 bytes. * A CPU can thus consume/refill/account against five different objcgs * (typically per-node variants of the same memcg) while incurring at * most one cache miss on the stock. */#define NR_OBJ_STOCK 5struct obj_stock_pcp { local_trylock_t lock; int8_t index; int16_t node_id; int16_t nr_slab_reclaimable_b; int16_t nr_slab_unreclaimable_b;#if PAGE_SHIFT > 16 /* * On rare archs with 256KiB base page size (hexagon and powerpc 44x) * keep nr_bytes to unsigned int as uint16_t cannot represent the full * sub-page remainder. Such archs are not cacheline optimization targets. */ unsigned int nr_bytes[NR_OBJ_STOCK];#else uint16_t nr_bytes[NR_OBJ_STOCK];#endif struct obj_cgroup *cached[NR_OBJ_STOCK];
struct work_struct work; unsigned long flags; uint8_t drain_idx;};
static DEFINE_PER_CPU_ALIGNED(struct obj_stock_pcp, obj_stock) = { .lock = INIT_LOCAL_TRYLOCK(lock), .index = -1, .node_id = NUMA_NO_NODE,};
static DEFINE_MUTEX(percpu_charge_mutex);
static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i);static void drain_obj_stock(struct obj_stock_pcp *stock);static bool obj_stock_flush_required(struct obj_stock_pcp *stock, struct mem_cgroup *root_memcg);
/** * consume_stock: Try to consume stocked charge on this cpu. * @memcg: memcg to consume from. * @nr_pages: how many pages to charge. * * Consume the cached charge if enough nr_pages are present otherwise return * failure. Also return failure for charge request larger than * MEMCG_CHARGE_BATCH or if the local lock is already taken. * * returns true if successful, false otherwise. */static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages){ struct memcg_stock_pcp *stock; uint8_t stock_pages; bool ret = false; int i;
if (nr_pages > MEMCG_CHARGE_BATCH || !local_trylock(&memcg_stock.lock)) return ret;
stock = this_cpu_ptr(&memcg_stock);
for (i = 0; i < NR_MEMCG_STOCK; ++i) { if (memcg != READ_ONCE(stock->cached[i])) continue;
stock_pages = READ_ONCE(stock->nr_pages[i]); if (stock_pages >= nr_pages) { stock_pages -= nr_pages; WRITE_ONCE(stock->nr_pages[i], stock_pages); if (!stock_pages) { WRITE_ONCE(stock->cached[i], NULL); css_put(&memcg->css); } ret = true; } break; }
local_unlock(&memcg_stock.lock);
return ret;}
static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages){ page_counter_uncharge(&memcg->memory, nr_pages); if (do_memsw_account()) page_counter_uncharge(&memcg->memsw, nr_pages);}
/* * Returns stocks cached in percpu and reset cached information. */static void drain_stock(struct memcg_stock_pcp *stock, int i){ struct mem_cgroup *old = READ_ONCE(stock->cached[i]); uint8_t stock_pages;
if (!old) return;
stock_pages = READ_ONCE(stock->nr_pages[i]); if (stock_pages) { memcg_uncharge(old, stock_pages); WRITE_ONCE(stock->nr_pages[i], 0); }
WRITE_ONCE(stock->cached[i], NULL); css_put(&old->css);}
static void drain_stock_fully(struct memcg_stock_pcp *stock){ int i;
for (i = 0; i < NR_MEMCG_STOCK; ++i) drain_stock(stock, i);}
static void drain_local_memcg_stock(struct work_struct *dummy){ struct memcg_stock_pcp *stock;
if (WARN_ONCE(!in_task(), "drain in non-task context")) return;
local_lock(&memcg_stock.lock);
stock = this_cpu_ptr(&memcg_stock); drain_stock_fully(stock); clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
local_unlock(&memcg_stock.lock);}
static void drain_local_obj_stock(struct work_struct *dummy){ struct obj_stock_pcp *stock;
if (WARN_ONCE(!in_task(), "drain in non-task context")) return;
local_lock(&obj_stock.lock);
stock = this_cpu_ptr(&obj_stock); drain_obj_stock(stock); clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
local_unlock(&obj_stock.lock);}
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages){ struct memcg_stock_pcp *stock; struct mem_cgroup *cached; unsigned int stock_pages; bool success = false; int empty_slot = -1; int i;
/* * nr_pages[] is a uint8_t and a slot's count is capped at * MEMCG_STOCK_HIGH. Raising MEMCG_CHARGE_BATCH beyond 127 would need * more careful handling of nr_pages[] in struct memcg_stock_pcp. */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX); BUILD_BUG_ON(MEMCG_STOCK_HIGH > U8_MAX);
VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg));
if (nr_pages > MEMCG_CHARGE_BATCH || !local_trylock(&memcg_stock.lock)) { /* * In case of larger than batch refill or unlikely failure to * lock the percpu memcg_stock.lock, uncharge memcg directly. */ memcg_uncharge(memcg, nr_pages); return; }
stock = this_cpu_ptr(&memcg_stock); for (i = 0; i < NR_MEMCG_STOCK; ++i) { cached = READ_ONCE(stock->cached[i]); if (!cached && empty_slot == -1) empty_slot = i; if (memcg == READ_ONCE(stock->cached[i])) { stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages; if (stock_pages > MEMCG_STOCK_HIGH) { memcg_uncharge(memcg, stock_pages - MEMCG_STOCK_LOW); stock_pages = MEMCG_STOCK_LOW; } WRITE_ONCE(stock->nr_pages[i], stock_pages); success = true; break; } }
if (!success) { i = empty_slot; if (i == -1) { i = stock->drain_idx++; if (stock->drain_idx == NR_MEMCG_STOCK) stock->drain_idx = 0; drain_stock(stock, i); } css_get(&memcg->css); WRITE_ONCE(stock->cached[i], memcg); WRITE_ONCE(stock->nr_pages[i], nr_pages); }
local_unlock(&memcg_stock.lock);}
static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock, struct mem_cgroup *root_memcg){ struct mem_cgroup *memcg; bool flush = false; int i;
rcu_read_lock(); for (i = 0; i < NR_MEMCG_STOCK; ++i) { memcg = READ_ONCE(stock->cached[i]); if (!memcg) continue;
if (READ_ONCE(stock->nr_pages[i]) && mem_cgroup_is_descendant(memcg, root_memcg)) { flush = true; break; } } rcu_read_unlock(); return flush;}
static bool schedule_drain_work(int cpu, struct work_struct *work){ /* * Protect housekeeping cpumask read and work enqueue together * in the same RCU critical section so that later cpuset isolated * partition update only need to wait for an RCU GP and flush the * pending work on newly isolated CPUs. */ guard(rcu)(); if (cpu_is_isolated(cpu)) return false;
queue_work_on(cpu, memcg_wq, work); return true;}
/* * Drains all per-CPU charge caches for given root_memcg resp. subtree * of the hierarchy under it. */void drain_all_stock(struct mem_cgroup *root_memcg){ int cpu, curcpu;
/* If someone's already draining, avoid adding running more workers. */ if (!mutex_trylock(&percpu_charge_mutex)) return; /* * Notify other cpus that system-wide "drain" is running * We do not care about races with the cpu hotplug because cpu down * as well as workers from this path always operate on the local * per-cpu data. CPU up doesn't touch memcg_stock at all. */ migrate_disable(); curcpu = smp_processor_id(); for_each_online_cpu(cpu) { struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu); struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu);
if (!test_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags) && is_memcg_drain_needed(memcg_st, root_memcg) && !test_and_set_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags)) { if (cpu == curcpu) drain_local_memcg_stock(&memcg_st->work); else if (!schedule_drain_work(cpu, &memcg_st->work)) clear_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags); }
if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) && obj_stock_flush_required(obj_st, root_memcg) && !test_and_set_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags)) { if (cpu == curcpu) drain_local_obj_stock(&obj_st->work); else if (!schedule_drain_work(cpu, &obj_st->work)) clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags); } } migrate_enable(); mutex_unlock(&percpu_charge_mutex);}
static int memcg_hotplug_cpu_dead(unsigned int cpu){ struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu); struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu);
/* no need for the local lock */ drain_obj_stock(obj_st); drain_stock_fully(memcg_st);
/* * A drain work queued before the CPU went away is executed by an * unbound worker on some other CPU and clears that CPU's flag, so * clear the flags here to make these stocks drainable again once * the CPU comes back online. */ clear_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags); clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags);
return 0;}
static unsigned long reclaim_high(struct mem_cgroup *memcg, unsigned int nr_pages, gfp_t gfp_mask){ unsigned long nr_reclaimed = 0;
do { unsigned long pflags;
if (page_counter_read(&memcg->memory) <= READ_ONCE(memcg->memory.high)) continue;
memcg_memory_event(memcg, MEMCG_HIGH);
psi_memstall_enter(&pflags); nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, MEMCG_RECLAIM_MAY_SWAP, NULL); psi_memstall_leave(&pflags); } while ((memcg = parent_mem_cgroup(memcg)) && !mem_cgroup_is_root(memcg));
return nr_reclaimed;}
static void high_work_func(struct work_struct *work){ struct mem_cgroup *memcg;
memcg = container_of(work, struct mem_cgroup, high_work); reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);}
static void high_irq_work_func(struct irq_work *work){ schedule_work(&container_of(work, struct mem_cgroup, high_irq_work)->high_work);}
/* * Clamp the maximum sleep time per allocation batch to 2 seconds. This is * enough to still cause a significant slowdown in most cases, while still * allowing diagnostics and tracing to proceed without becoming stuck. */#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
/* * When calculating the delay, we use these either side of the exponentiation to * maintain precision and scale to a reasonable number of jiffies (see the table * below. * * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the * overage ratio to a delay. * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the * proposed penalty in order to reduce to a reasonable number of jiffies, and * to produce a reasonable delay curve. * * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a * reasonable delay curve compared to precision-adjusted overage, not * penalising heavily at first, but still making sure that growth beyond the * limit penalises misbehaviour cgroups by slowing them down exponentially. For * example, with a high of 100 megabytes: * * +-------+------------------------+ * | usage | time to allocate in ms | * +-------+------------------------+ * | 100M | 0 | * | 101M | 6 | * | 102M | 25 | * | 103M | 57 | * | 104M | 102 | * | 105M | 159 | * | 106M | 230 | * | 107M | 313 | * | 108M | 409 | * | 109M | 518 | * | 110M | 639 | * | 111M | 774 | * | 112M | 921 | * | 113M | 1081 | * | 114M | 1254 | * | 115M | 1439 | * | 116M | 1638 | * | 117M | 1849 | * | 118M | 2000 | * | 119M | 2000 | * | 120M | 2000 | * +-------+------------------------+ */ #define MEMCG_DELAY_PRECISION_SHIFT 20 #define MEMCG_DELAY_SCALING_SHIFT 14
static u64 calculate_overage(unsigned long usage, unsigned long high){ u64 overage;
if (usage <= high) return 0;
/* * Prevent division by 0 in overage calculation by acting as if * it was a threshold of 1 page */ high = max(high, 1UL);
overage = usage - high; overage <<= MEMCG_DELAY_PRECISION_SHIFT; return div64_u64(overage, high);}
static u64 mem_find_max_overage(struct mem_cgroup *memcg){ u64 overage, max_overage = 0;
do { overage = calculate_overage(page_counter_read(&memcg->memory), READ_ONCE(memcg->memory.high)); max_overage = max(overage, max_overage); } while ((memcg = parent_mem_cgroup(memcg)) && !mem_cgroup_is_root(memcg));
return max_overage;}
static u64 swap_find_max_overage(struct mem_cgroup *memcg){ u64 overage, max_overage = 0;
do { overage = calculate_overage(page_counter_read(&memcg->swap), READ_ONCE(memcg->swap.high)); if (overage) memcg_memory_event(memcg, MEMCG_SWAP_HIGH); max_overage = max(overage, max_overage); } while ((memcg = parent_mem_cgroup(memcg)) && !mem_cgroup_is_root(memcg));
return max_overage;}
/* * Get the number of jiffies that we should penalise a mischievous cgroup which * is exceeding its memory.high by checking both it and its ancestors. */static unsigned long calculate_high_delay(unsigned int nr_pages, u64 max_overage){ unsigned long penalty_jiffies;
if (!max_overage) return 0;
/* * We use overage compared to memory.high to calculate the number of * jiffies to sleep (penalty_jiffies). Ideally this value should be * fairly lenient on small overages, and increasingly harsh when the * memcg in question makes it clear that it has no intention of stopping * its crazy behaviour, so we exponentially increase the delay based on * overage amount. */ penalty_jiffies = max_overage * max_overage * HZ; penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT; penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
/* * Factor in the task's own contribution to the overage, such that four * N-sized allocations are throttled approximately the same as one * 4N-sized allocation. * * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or * larger the current charge patch is than that. */ return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;}
/* * Reclaims memory over the high limit. Called directly from * try_charge() (context permitting), as well as from the userland * return path where reclaim is always able to block. */void __mem_cgroup_handle_over_high(gfp_t gfp_mask){ unsigned long penalty_jiffies; unsigned long pflags; unsigned long nr_reclaimed; unsigned int nr_pages = current->memcg_nr_pages_over_high; int nr_retries = MAX_RECLAIM_RETRIES; struct mem_cgroup *memcg; bool in_retry = false;
memcg = get_mem_cgroup_from_mm(current->mm); current->memcg_nr_pages_over_high = 0;
retry_reclaim: /* * Bail if the task is already exiting. Unlike memory.max, * memory.high enforcement isn't as strict, and there is no * OOM killer involved, which means the excess could already * be much bigger (and still growing) than it could for * memory.max; the dying task could get stuck in fruitless * reclaim for a long time, which isn't desirable. */ if (task_is_dying()) goto out;
/* * The allocating task should reclaim at least the batch size, but for * subsequent retries we only want to do what's necessary to prevent oom * or breaching resource isolation. * * This is distinct from memory.max or page allocator behaviour because * memory.high is currently batched, whereas memory.max and the page * allocator run every time an allocation is made. */ nr_reclaimed = reclaim_high(memcg, in_retry ? SWAP_CLUSTER_MAX : nr_pages, gfp_mask);
/* * memory.high is breached and reclaim is unable to keep up. Throttle * allocators proactively to slow down excessive growth. */ penalty_jiffies = calculate_high_delay(nr_pages, mem_find_max_overage(memcg));
penalty_jiffies += calculate_high_delay(nr_pages, swap_find_max_overage(memcg));
/* * Clamp the max delay per usermode return so as to still keep the * application moving forwards and also permit diagnostics, albeit * extremely slowly. */ penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
/* * Don't sleep if the amount of jiffies this memcg owes us is so low * that it's not even worth doing, in an attempt to be nice to those who * go only a small amount over their memory.high value and maybe haven't * been aggressively reclaimed enough yet. */ if (penalty_jiffies <= HZ / 100) goto out;
/* * If reclaim is making forward progress but we're still over * memory.high, we want to encourage that rather than doing allocator * throttling. */ if (nr_reclaimed || nr_retries--) { in_retry = true; goto retry_reclaim; }
/* * Reclaim didn't manage to push usage below the limit, slow * this allocating task down. * * If we exit early, we're guaranteed to die (since * schedule_timeout_killable sets TASK_KILLABLE). This means we don't * need to account for any ill-begotten jiffies to pay them off later. */ psi_memstall_enter(&pflags); schedule_timeout_killable(penalty_jiffies); psi_memstall_leave(&pflags);
out: css_put(&memcg->css);}
static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, unsigned int nr_pages){ unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages); int nr_retries = MAX_RECLAIM_RETRIES; struct mem_cgroup *mem_over_limit; struct page_counter *counter; unsigned long nr_reclaimed; bool passed_oom = false; unsigned int reclaim_options; bool drained = false; bool raised_max_event = false; unsigned long pflags; bool allow_spinning = gfpflags_allow_spinning(gfp_mask); int ret = 0;
retry: if (consume_stock(memcg, nr_pages)) return ret;
if (!allow_spinning) /* Avoid the refill and flush of the older stock */ batch = nr_pages;
reclaim_options = MEMCG_RECLAIM_MAY_SWAP; if (!do_memsw_account() || page_counter_try_charge(&memcg->memsw, batch, &counter)) { if (page_counter_try_charge(&memcg->memory, batch, &counter)) goto done_restock; if (do_memsw_account()) page_counter_uncharge(&memcg->memsw, batch); mem_over_limit = mem_cgroup_from_counter(counter, memory); } else { mem_over_limit = mem_cgroup_from_counter(counter, memsw); reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP; }
if (batch > nr_pages) { batch = nr_pages; goto retry; }
/* * Prevent unbounded recursion when reclaim operations need to * allocate memory. This might exceed the limits temporarily, * but we prefer facilitating memory reclaim and getting back * under the limit over triggering OOM kills in these cases. */ if (unlikely(current->flags & PF_MEMALLOC)) goto force;
if (unlikely(task_in_memcg_oom(current))) goto nomem;
if (!gfpflags_allow_blocking(gfp_mask)) goto nomem;
/* * OOM victim still needs to charge memory to exit. OOM reaper should * help but it might fail on mmap_lock contention. If the victim is a * large thread group then all exiting threads might compete on oom_lock * just to learn that there is nothing really killable anymore. Bail * out early and fail the charge to expedite their exit. They are * considered fully reclaimed by the oom reaper and they shouldn't * contribute further charges. */ if (tsk_is_oom_victim(current) && mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm)) goto nomem;
__memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); raised_max_event = true;
psi_memstall_enter(&pflags); nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, gfp_mask, reclaim_options, NULL); psi_memstall_leave(&pflags);
if (mem_cgroup_margin(mem_over_limit) >= nr_pages) goto retry;
if (!drained) { drain_all_stock(mem_over_limit); drained = true; goto retry; }
if (gfp_mask & __GFP_NORETRY) goto nomem; /* * Even though the limit is exceeded at this point, reclaim * may have been able to free some pages. Retry the charge * before killing the task. * * Only for regular pages, though: huge pages are rather * unlikely to succeed so close to the limit, and we fall back * to regular pages anyway in case of failure. */ if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) goto retry;
if (nr_retries--) goto retry;
if (gfp_mask & __GFP_RETRY_MAYFAIL) goto nomem;
/* Avoid endless loop for tasks bypassed by the oom killer */ if (passed_oom && task_is_dying()) goto nomem;
/* * keep retrying as long as the memcg oom killer is able to make * a forward progress or bypass the charge if the oom killer * couldn't make any progress. */ if (mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(nr_pages * PAGE_SIZE))) { passed_oom = true; nr_retries = MAX_RECLAIM_RETRIES; goto retry; }nomem: /* * Memcg doesn't have a dedicated reserve for atomic * allocations. But like the global atomic pool, we need to * put the burden of reclaim on regular allocation requests * and let these go through as privileged allocations. */ if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) { ret = -ENOMEM; goto out; }force: /* * The allocation either can't fail or will lead to more memory * being freed very soon. Allow memory usage go over the limit * temporarily by force charging it. */ page_counter_charge(&memcg->memory, nr_pages); if (do_memsw_account()) page_counter_charge(&memcg->memsw, nr_pages);
out: /* * Don't forget to raise a MEMCG_MAX event for forced or rejected * requests. */ if (!raised_max_event) __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
return ret;
done_restock: if (batch > nr_pages) refill_stock(memcg, batch - nr_pages);
/* * If the hierarchy is above the normal consumption range, schedule * reclaim on returning to userland. We can perform reclaim here * if __GFP_RECLAIM but let's always punt for simplicity and so that * GFP_KERNEL can consistently be used during reclaim. @memcg is * not recorded as it most likely matches current's and won't * change in the meantime. As high limit is checked again before * reclaim, the cost of mismatch is negligible. */ do { bool mem_high, swap_high;
mem_high = page_counter_read(&memcg->memory) > READ_ONCE(memcg->memory.high); swap_high = page_counter_read(&memcg->swap) > READ_ONCE(memcg->swap.high);
/* Don't bother a random interrupted task */ if (!in_task()) { if (mem_high) { if (allow_spinning) schedule_work(&memcg->high_work); else irq_work_queue(&memcg->high_irq_work); break; } continue; }
if (mem_high || swap_high) { /* * The allocating tasks in this cgroup will need to do * reclaim or be throttled to prevent further growth * of the memory or swap footprints. * * Target some best-effort fairness between the tasks, * and distribute reclaim work and delay penalties * based on how much each task is actually allocating. */ current->memcg_nr_pages_over_high += batch; set_notify_resume(current); break; } } while ((memcg = parent_mem_cgroup(memcg)));
/* * Reclaim is set up above to be called from the userland * return path. But also attempt synchronous reclaim to avoid * excessive overrun while the task is still inside the * kernel. If this is successful, the return path will see it * when it rechecks the overage and simply bail out. */ if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH && !(current->flags & PF_MEMALLOC) && gfpflags_allow_blocking(gfp_mask)) __mem_cgroup_handle_over_high(gfp_mask); return ret;}
static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, unsigned int nr_pages){ if (mem_cgroup_is_root(memcg)) return 0;
return try_charge_memcg(memcg, gfp_mask, nr_pages);}
static void commit_charge(struct folio *folio, struct obj_cgroup *objcg){ VM_BUG_ON_FOLIO(folio_memcg_charged(folio), folio); /* * Any of the following ensures folio's objcg stability: * * - the page lock * - LRU isolation * - exclusive reference */ folio->memcg_data = (unsigned long)objcg;}
#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMICstatic inline void account_slab_nmi_safe(struct mem_cgroup *memcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr){ struct lruvec *lruvec;
if (likely(!in_nmi())) { lruvec = mem_cgroup_lruvec(memcg, pgdat); mod_memcg_lruvec_state(lruvec, idx, nr); } else { struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
/* preemption is disabled in_nmi(). */ __css_rstat_updated(&memcg->css, smp_processor_id()); if (idx == NR_SLAB_RECLAIMABLE_B) atomic_add(nr, &pn->slab_reclaimable); else atomic_add(nr, &pn->slab_unreclaimable); }}#elsestatic inline void account_slab_nmi_safe(struct mem_cgroup *memcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr){ struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat); mod_memcg_lruvec_state(lruvec, idx, nr);}#endif
static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr){ struct mem_cgroup *memcg;
rcu_read_lock(); memcg = obj_cgroup_memcg(objcg); account_slab_nmi_safe(memcg, pgdat, idx, nr); rcu_read_unlock();}
static __always_inlinestruct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p){ /* * Slab objects are accounted individually, not per-page. * Memcg membership data for each individual object is saved in * slab->obj_exts. */ unsigned long obj_exts; struct slabobj_ext *obj_ext; struct obj_cgroup *objcg;
obj_exts = slab_obj_exts(slab); if (!obj_exts) return NULL;
if (!slab_needs_objcg(slab)) return NULL;
get_slab_obj_exts(obj_exts); obj_ext = slab_obj_ext(slab->slab_cache, slab, obj_exts, p); objcg = slab_obj_ext_objcg(slab, obj_ext); if (objcg) { put_slab_obj_exts(obj_exts); return obj_cgroup_memcg(objcg); } put_slab_obj_exts(obj_exts);
return NULL;}
/* * Returns a pointer to the memory cgroup to which the kernel object is charged. * It is not suitable for objects allocated using vmalloc(). * * A passed kernel object must be a slab object or a generic kernel page. * * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(), * cgroup_mutex, etc. */struct mem_cgroup *mem_cgroup_from_virt(void *p){ struct slab *slab;
if (mem_cgroup_disabled()) return NULL;
slab = virt_to_slab(p); if (slab) return mem_cgroup_from_obj_slab(slab, p); return folio_memcg_check(virt_to_folio(p));}
static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg, int nid){ for (; memcg; memcg = parent_mem_cgroup(memcg)) { struct obj_cgroup *objcg = rcu_dereference(memcg->nodeinfo[nid]->objcg);
if (likely(objcg && obj_cgroup_tryget(objcg))) return objcg; }
return NULL;}
static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg, int nid){ struct obj_cgroup *objcg;
rcu_read_lock(); objcg = __get_obj_cgroup_from_memcg(memcg, nid); rcu_read_unlock();
return objcg;}
static struct obj_cgroup *current_objcg_update(void){ struct mem_cgroup *memcg; struct obj_cgroup *old, *objcg = NULL;
do { /* Atomically drop the update bit. */ old = xchg(¤t->objcg, NULL); if (old) { old = (struct obj_cgroup *) ((unsigned long)old & ~CURRENT_OBJCG_UPDATE_FLAG); obj_cgroup_put(old);
old = NULL; }
/* If new objcg is NULL, no reason for the second atomic update. */ if (!current->mm || (current->flags & PF_KTHREAD)) return NULL;
/* * Release the objcg pointer from the previous iteration, * if try_cmpxcg() below fails. */ if (unlikely(objcg)) { obj_cgroup_put(objcg); objcg = NULL; }
/* * Obtain the new objcg pointer. The current task can be * asynchronously moved to another memcg and the previous * memcg can be offlined. So let's get the memcg pointer * and try get a reference to objcg under a rcu read lock. */
rcu_read_lock(); memcg = mem_cgroup_from_task(current); objcg = __get_obj_cgroup_from_memcg(memcg, numa_node_id()); rcu_read_unlock();
/* * Try set up a new objcg pointer atomically. If it * fails, it means the update flag was set concurrently, so * the whole procedure should be repeated. */ } while (!try_cmpxchg(¤t->objcg, &old, objcg));
return objcg;}
__always_inline struct obj_cgroup *current_obj_cgroup(void){ struct mem_cgroup *memcg; struct obj_cgroup *objcg; int nid = numa_node_id();
if (IS_ENABLED(CONFIG_MEMCG_NMI_UNSAFE) && in_nmi()) return NULL;
if (in_task()) { memcg = current->active_memcg; if (unlikely(memcg)) goto from_memcg;
objcg = READ_ONCE(current->objcg); if (unlikely((unsigned long)objcg & CURRENT_OBJCG_UPDATE_FLAG)) objcg = current_objcg_update(); /* * Objcg reference is kept by the task, so it's safe * to use the objcg by the current task. */ return objcg ? : rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1); }
memcg = this_cpu_read(int_active_memcg); if (unlikely(memcg)) goto from_memcg;
return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1);
from_memcg: for (; memcg; memcg = parent_mem_cgroup(memcg)) { /* * Memcg pointer is protected by scope (see set_active_memcg()) * and is pinning the corresponding objcg, so objcg can't go * away and can be used within the scope without any additional * protection. */ objcg = rcu_dereference_check(memcg->nodeinfo[nid]->objcg, 1); if (likely(objcg)) return objcg; }
return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1);}
struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio){ struct obj_cgroup *objcg;
objcg = folio_objcg(folio); if (objcg) obj_cgroup_get(objcg);
return objcg;}
#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMICstatic inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val){ if (likely(!in_nmi())) { mod_memcg_state(memcg, MEMCG_KMEM, val); } else { /* preemption is disabled in_nmi(). */ __css_rstat_updated(&memcg->css, smp_processor_id()); atomic_add(val, &memcg->kmem_stat); }}#elsestatic inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val){ mod_memcg_state(memcg, MEMCG_KMEM, val);}#endif
/* * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg * @objcg: object cgroup to uncharge * @nr_pages: number of pages to uncharge */static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, unsigned int nr_pages){ struct mem_cgroup *memcg;
memcg = get_mem_cgroup_from_objcg(objcg);
account_kmem_nmi_safe(memcg, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); if (!mem_cgroup_is_root(memcg)) refill_stock(memcg, nr_pages);
css_put(&memcg->css);}
/* * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg * @objcg: object cgroup to charge * @gfp: reclaim mode * @nr_pages: number of pages to charge * * Returns 0 on success, an error code on failure. */static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, unsigned int nr_pages){ struct mem_cgroup *memcg; int ret;
memcg = get_mem_cgroup_from_objcg(objcg);
ret = try_charge(memcg, gfp, nr_pages); if (ret) goto out;
account_kmem_nmi_safe(memcg, nr_pages); memcg1_account_kmem(memcg, nr_pages);out: css_put(&memcg->css);
return ret;}
static struct obj_cgroup *page_objcg(const struct page *page){ unsigned long memcg_data = page->memcg_data;
if (mem_cgroup_disabled() || !memcg_data) return NULL;
VM_BUG_ON_PAGE((memcg_data & OBJEXTS_FLAGS_MASK) != MEMCG_DATA_KMEM, page); return (struct obj_cgroup *)(memcg_data - MEMCG_DATA_KMEM);}
static void page_set_objcg(struct page *page, const struct obj_cgroup *objcg){ page->memcg_data = (unsigned long)objcg | MEMCG_DATA_KMEM;}
/** * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup * @page: page to charge * @gfp: reclaim mode * @order: allocation order * * Returns 0 on success, an error code on failure. */int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order){ struct obj_cgroup *objcg; int ret = 0;
objcg = current_obj_cgroup(); if (objcg && !obj_cgroup_is_root(objcg)) { ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order); if (!ret) { obj_cgroup_get(objcg); page_set_objcg(page, objcg); return 0; } } return ret;}
/** * __memcg_kmem_uncharge_page: uncharge a kmem page * @page: page to uncharge * @order: allocation order */void __memcg_kmem_uncharge_page(struct page *page, int order){ struct obj_cgroup *objcg = page_objcg(page); unsigned int nr_pages = 1 << order;
if (!objcg) return;
obj_cgroup_uncharge_pages(objcg, nr_pages); page->memcg_data = 0; obj_cgroup_put(objcg);}
static struct obj_stock_pcp *trylock_stock(void){ if (local_trylock(&obj_stock.lock)) return this_cpu_ptr(&obj_stock);
return NULL;}
static void unlock_stock(struct obj_stock_pcp *stock){ if (stock) local_unlock(&obj_stock.lock);}
/* Call after __refill_obj_stock() so a slot for objcg exists in the stock */static void __account_obj_stock(struct obj_cgroup *objcg, struct obj_stock_pcp *stock, int nr, struct pglist_data *pgdat, enum node_stat_item idx){ int16_t *bytes; int i;
/* * Though at the moment MAX_NUMNODES <= 1024 in all archs but let's make * sure it does not exceed S16_MAX otherwise we need to fix node_id type * in struct obj_stock_pcp. */ BUILD_BUG_ON(MAX_NUMNODES >= S16_MAX);
if (!stock) goto direct;
for (i = 0; i < NR_OBJ_STOCK; ++i) { if (READ_ONCE(stock->cached[i]) == objcg) break; } if (i == NR_OBJ_STOCK) goto direct;
/* * Save vmstat data in stock and skip vmstat array update unless * accumulating over a page of vmstat data or when the objcg slot or * pgdat the stats belong to changes. */ if (stock->index < 0) { stock->index = i; stock->node_id = pgdat->node_id; } else if (stock->index != i || stock->node_id != pgdat->node_id) { struct obj_cgroup *old = READ_ONCE(stock->cached[stock->index]); struct pglist_data *oldpg = NODE_DATA(stock->node_id);
if (stock->nr_slab_reclaimable_b) { mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b = 0; } if (stock->nr_slab_unreclaimable_b) { mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b = 0; } stock->index = i; stock->node_id = pgdat->node_id; }
bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b : &stock->nr_slab_unreclaimable_b;
/* * Fold @nr into the cached value and decide whether to keep it cached * or flush it directly. Cache the combined value when it fits in the * int16_t storage and either the cache was empty (so even a value * above PAGE_SIZE gets a chance to be canceled by a paired delta) or * the combined value is within the PAGE_SIZE flush threshold. */ nr += *bytes; if (abs(nr) <= S16_MAX && (!*bytes || abs(nr) <= PAGE_SIZE)) { *bytes = nr; nr = 0; } else { *bytes = 0; }direct: if (nr) mod_objcg_mlstate(objcg, pgdat, idx, nr);}
static bool __consume_obj_stock(struct obj_cgroup *objcg, struct obj_stock_pcp *stock, unsigned int nr_bytes){ int i;
for (i = 0; i < NR_OBJ_STOCK; ++i) { if (READ_ONCE(stock->cached[i]) != objcg) continue; if (stock->nr_bytes[i] >= nr_bytes) { stock->nr_bytes[i] -= nr_bytes; return true; } return false; }
return false;}
static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes){ struct obj_stock_pcp *stock; bool ret = false;
stock = trylock_stock(); if (!stock) return ret;
ret = __consume_obj_stock(objcg, stock, nr_bytes); unlock_stock(stock);
return ret;}
/* Flush the cached slab stats (if any) back to their owning objcg/pgdat. */static void drain_obj_stock_stats(struct obj_stock_pcp *stock){ struct obj_cgroup *old; struct pglist_data *oldpg;
if (stock->index < 0) return;
old = READ_ONCE(stock->cached[stock->index]); oldpg = NODE_DATA(stock->node_id);
if (stock->nr_slab_reclaimable_b) { mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b = 0; } if (stock->nr_slab_unreclaimable_b) { mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b = 0; } stock->index = -1; stock->node_id = NUMA_NO_NODE;}
static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i){ struct obj_cgroup *old = READ_ONCE(stock->cached[i]);
if (!old) return;
if (stock->nr_bytes[i]) { unsigned int nr_pages = stock->nr_bytes[i] >> PAGE_SHIFT; unsigned int nr_bytes = stock->nr_bytes[i] & (PAGE_SIZE - 1);
if (nr_pages) { struct mem_cgroup *memcg;
memcg = get_mem_cgroup_from_objcg(old);
memcg_uncharge_kmem(memcg, nr_pages);
css_put(&memcg->css); }
/* * The leftover is flushed to the centralized per-memcg value. * On the next attempt to refill obj stock it will be moved * to a per-cpu stock (probably, on an other CPU), see * refill_obj_stock(). * * How often it's flushed is a trade-off between the memory * limit enforcement accuracy and potential CPU contention, * so it might be changed in the future. */ atomic_add(nr_bytes, &old->nr_charged_bytes); stock->nr_bytes[i] = 0; }
/* Flush vmstat data when its owning slot is being drained. */ if (stock->index == i) drain_obj_stock_stats(stock);
WRITE_ONCE(stock->cached[i], NULL); obj_cgroup_put(old);}
static void drain_obj_stock(struct obj_stock_pcp *stock){ int i;
for (i = 0; i < NR_OBJ_STOCK; ++i) drain_obj_stock_slot(stock, i);}
static bool obj_stock_flush_required(struct obj_stock_pcp *stock, struct mem_cgroup *root_memcg){ struct obj_cgroup *objcg; struct mem_cgroup *memcg; bool flush = false; int i;
rcu_read_lock(); for (i = 0; i < NR_OBJ_STOCK; ++i) { objcg = READ_ONCE(stock->cached[i]); if (!objcg) continue; memcg = obj_cgroup_memcg(objcg); if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) { flush = true; break; } } rcu_read_unlock();
return flush;}
static void __refill_obj_stock(struct obj_cgroup *objcg, struct obj_stock_pcp *stock, unsigned int nr_bytes, bool allow_uncharge){ unsigned int nr_pages = 0; unsigned int stock_nr_bytes; int i, slot = -1, empty_slot = -1;
if (!stock) { nr_pages = nr_bytes >> PAGE_SHIFT; nr_bytes = nr_bytes & (PAGE_SIZE - 1); atomic_add(nr_bytes, &objcg->nr_charged_bytes); goto out; }
for (i = 0; i < NR_OBJ_STOCK; ++i) { struct obj_cgroup *cached = READ_ONCE(stock->cached[i]);
if (!cached) { if (empty_slot == -1) empty_slot = i; continue; } if (cached == objcg) { slot = i; break; } }
if (slot == -1) { slot = empty_slot; if (slot == -1) { slot = stock->drain_idx++; if (stock->drain_idx == NR_OBJ_STOCK) stock->drain_idx = 0; drain_obj_stock_slot(stock, slot); } obj_cgroup_get(objcg); /* * Keep the xchg result in the unsigned int local; storing * it directly into stock->nr_bytes[slot] (uint16_t) would * silently truncate values >= U16_MAX and bypass the flush * guard below, leaking page-counter charges. */ stock_nr_bytes = atomic_read(&objcg->nr_charged_bytes) ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; WRITE_ONCE(stock->cached[slot], objcg);
allow_uncharge = true; /* Allow uncharge when objcg changes */ } else { stock_nr_bytes = stock->nr_bytes[slot]; }
stock_nr_bytes += nr_bytes;
if ((allow_uncharge && (stock_nr_bytes > PAGE_SIZE)) || stock_nr_bytes > U16_MAX) { nr_pages = stock_nr_bytes >> PAGE_SHIFT; stock_nr_bytes &= (PAGE_SIZE - 1); } stock->nr_bytes[slot] = stock_nr_bytes;
out: if (nr_pages) obj_cgroup_uncharge_pages(objcg, nr_pages);}
static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, bool allow_uncharge){ struct obj_stock_pcp *stock = trylock_stock(); __refill_obj_stock(objcg, stock, nr_bytes, allow_uncharge); unlock_stock(stock);}
static int __obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size, size_t *remainder){ size_t charge_size; int ret;
charge_size = PAGE_ALIGN(size); ret = obj_cgroup_charge_pages(objcg, gfp, charge_size >> PAGE_SHIFT); if (!ret) *remainder = charge_size - size;
return ret;}
int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size){ size_t remainder; int ret;
if (likely(consume_obj_stock(objcg, size))) return 0;
/* * In theory, objcg->nr_charged_bytes can have enough * pre-charged bytes to satisfy the allocation. However, * flushing objcg->nr_charged_bytes requires two atomic * operations, and objcg->nr_charged_bytes can't be big. * The shared objcg->nr_charged_bytes can also become a * performance bottleneck if all tasks of the same memcg are * trying to update it. So it's better to ignore it and try * grab some new pages. The stock's nr_bytes will be flushed to * objcg->nr_charged_bytes later on when objcg changes. * * The stock's nr_bytes may contain enough pre-charged bytes * to allow one less page from being charged, but we can't rely * on the pre-charged bytes not being changed outside of * consume_obj_stock() or refill_obj_stock(). So ignore those * pre-charged bytes as well when charging pages. To avoid a * page uncharge right after a page charge, we set the * allow_uncharge flag to false when calling refill_obj_stock() * to temporarily allow the pre-charged bytes to exceed the page * size limit. The maximum reachable value of the pre-charged * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data * race. */ ret = __obj_cgroup_charge(objcg, gfp, size, &remainder); if (!ret && remainder) refill_obj_stock(objcg, remainder, false);
return ret;}
void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size){ refill_obj_stock(objcg, size, true);}
static inline size_t obj_full_size(struct kmem_cache *s){ /* * For each accounted object there is an extra space which is used * to store obj_cgroup membership. Charge it too. */ return s->size + sizeof(struct obj_cgroup *);}
bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, gfp_t flags, unsigned int slab_alloc_flags, size_t size, void **p){ size_t obj_size = obj_full_size(s); struct obj_cgroup *objcg; struct slab *slab; size_t i;
/* * The obtained objcg pointer is safe to use within the current scope, * defined by current task or set_active_memcg() pair. * obj_cgroup_get() is used to get a permanent reference. */ objcg = current_obj_cgroup(); if (!objcg || obj_cgroup_is_root(objcg)) return true;
/* * slab_alloc_node() avoids the NULL check, so we might be called with a * single NULL object. kmem_cache_alloc_bulk() aborts if it can't fill * the whole requested size. * return success as there's nothing to free back */ if (unlikely(*p == NULL)) return true;
flags &= gfp_allowed_mask;
if (lru) { int ret; struct mem_cgroup *memcg;
memcg = get_mem_cgroup_from_objcg(objcg); ret = memcg_list_lru_alloc(memcg, lru, flags); css_put(&memcg->css);
if (ret) return false; }
for (i = 0; i < size; i++) { unsigned long obj_exts; struct slabobj_ext *obj_ext; struct obj_stock_pcp *stock;
slab = virt_to_slab(p[i]);
if (!slab_obj_exts(slab)) { if (is_kfence_address(p[i])) continue; if (alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) continue; }
/* * if we fail and size is 1, memcg_alloc_abort_single() will * just free the object, which is ok as we have not assigned * objcg to its obj_ext yet * * for larger sizes, kmem_cache_free_bulk() will uncharge * any objects that were already charged and obj_ext assigned * * TODO: we could batch this until slab_pgdat(slab) changes * between iterations, with a more complicated undo */ stock = trylock_stock(); if (!stock || !__consume_obj_stock(objcg, stock, obj_size)) { size_t remainder;
unlock_stock(stock); if (__obj_cgroup_charge(objcg, flags, obj_size, &remainder)) return false; stock = trylock_stock(); if (remainder) __refill_obj_stock(objcg, stock, remainder, false); } __account_obj_stock(objcg, stock, obj_size, slab_pgdat(slab), cache_vmstat_idx(s)); unlock_stock(stock);
obj_exts = slab_obj_exts(slab); get_slab_obj_exts(obj_exts); obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]);
obj_cgroup_get(objcg); slab_obj_ext_set_objcg(slab, obj_ext, objcg);
put_slab_obj_exts(obj_exts); }
return true;}
void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects, unsigned long obj_exts){ size_t obj_size = obj_full_size(s);
for (int i = 0; i < objects; i++) { struct obj_cgroup *objcg; struct slabobj_ext *obj_ext; struct obj_stock_pcp *stock;
obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]); objcg = slab_obj_ext_objcg(slab, obj_ext); if (!objcg) continue;
slab_obj_ext_set_objcg(slab, obj_ext, NULL);
stock = trylock_stock(); __refill_obj_stock(objcg, stock, obj_size, true); __account_obj_stock(objcg, stock, -obj_size, slab_pgdat(slab), cache_vmstat_idx(s)); unlock_stock(stock);
obj_cgroup_put(objcg); }}
/* * The objcg is only set on the first page, so transfer it to all the * other pages. */void split_page_memcg(struct page *page, unsigned order){ struct obj_cgroup *objcg = page_objcg(page); unsigned int i, nr = 1 << order;
if (!objcg) return;
for (i = 1; i < nr; i++) page_set_objcg(&page[i], objcg);
obj_cgroup_get_many(objcg, nr - 1);}
void folio_split_memcg_refs(struct folio *folio, unsigned old_order, unsigned new_order){ unsigned new_refs;
if (mem_cgroup_disabled() || !folio_memcg_charged(folio)) return;
new_refs = (1 << (old_order - new_order)) - 1; obj_cgroup_get_many(folio_objcg(folio), new_refs);}
static void memcg_online_kmem(struct mem_cgroup *memcg){ if (mem_cgroup_kmem_disabled()) return;
if (unlikely(mem_cgroup_is_root(memcg))) return;
static_branch_enable(&memcg_kmem_online_key);
memcg->kmemcg_id = memcg->private_id;}
static void memcg_offline_kmem(struct mem_cgroup *memcg){ struct mem_cgroup *parent;
if (mem_cgroup_kmem_disabled()) return;
if (unlikely(mem_cgroup_is_root(memcg))) return;
parent = parent_mem_cgroup(memcg); memcg_reparent_list_lrus(memcg, parent);}
#ifdef CONFIG_CGROUP_WRITEBACK
#include <trace/events/writeback.h>
static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp){ return wb_domain_init(&memcg->cgwb_domain, gfp);}
static void memcg_wb_domain_exit(struct mem_cgroup *memcg){ wb_domain_exit(&memcg->cgwb_domain);}
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg){ wb_domain_size_changed(&memcg->cgwb_domain);}
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb){ struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
if (!memcg->css.parent) return NULL;
return &memcg->cgwb_domain;}
/** * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg * @wb: bdi_writeback in question * @pfilepages: out parameter for number of file pages * @pheadroom: out parameter for number of allocatable pages according to memcg * @pdirty: out parameter for number of dirty pages * @pwriteback: out parameter for number of pages under writeback * * Determine the numbers of file, headroom, dirty, and writeback pages in * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom * is a bit more involved. * * A memcg's headroom is "min(max, high) - used". In the hierarchy, the * headroom is calculated as the lowest headroom of itself and the * ancestors. Note that this doesn't consider the actual amount of * available memory in the system. The caller should further cap * *@pheadroom accordingly. */void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, unsigned long *pheadroom, unsigned long *pdirty, unsigned long *pwriteback){ struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); struct mem_cgroup *parent;
mem_cgroup_flush_stats_ratelimited(memcg);
*pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) + memcg_page_state(memcg, NR_ACTIVE_FILE);
*pheadroom = PAGE_COUNTER_MAX; while ((parent = parent_mem_cgroup(memcg))) { unsigned long ceiling = min(READ_ONCE(memcg->memory.max), READ_ONCE(memcg->memory.high)); unsigned long used = page_counter_read(&memcg->memory);
*pheadroom = min(*pheadroom, ceiling - min(ceiling, used)); memcg = parent; }}
/* * Foreign dirty flushing * * There's an inherent mismatch between memcg and writeback. The former * tracks ownership per-page while the latter per-inode. This was a * deliberate design decision because honoring per-page ownership in the * writeback path is complicated, may lead to higher CPU and IO overheads * and deemed unnecessary given that write-sharing an inode across * different cgroups isn't a common use-case. * * Combined with inode majority-writer ownership switching, this works well * enough in most cases but there are some pathological cases. For * example, let's say there are two cgroups A and B which keep writing to * different but confined parts of the same inode. B owns the inode and * A's memory is limited far below B's. A's dirty ratio can rise enough to * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid * triggering background writeback. A will be slowed down without a way to * make writeback of the dirty pages happen. * * Conditions like the above can lead to a cgroup getting repeatedly and * severely throttled after making some progress after each * dirty_expire_interval while the underlying IO device is almost * completely idle. * * Solving this problem completely requires matching the ownership tracking * granularities between memcg and writeback in either direction. However, * the more egregious behaviors can be avoided by simply remembering the * most recent foreign dirtying events and initiating remote flushes on * them when local writeback isn't enough to keep the memory clean enough. * * The following two functions implement such mechanism. When a foreign * page - a page whose memcg and writeback ownerships don't match - is * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning * bdi_writeback on the page owning memcg. When balance_dirty_pages() * decides that the memcg needs to sleep due to high dirty ratio, it calls * mem_cgroup_flush_foreign() which queues writeback on the recorded * foreign bdi_writebacks which haven't expired. Both the numbers of * recorded bdi_writebacks and concurrent in-flight foreign writebacks are * limited to MEMCG_CGWB_FRN_CNT. * * The mechanism only remembers IDs and doesn't hold any object references. * As being wrong occasionally doesn't matter, updates and accesses to the * records are lockless and racy. */void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio, struct bdi_writeback *wb){ struct mem_cgroup *memcg = folio_memcg(folio); struct memcg_cgwb_frn *frn; u64 now = get_jiffies_64(); u64 oldest_at = now; int oldest = -1; int i;
trace_track_foreign_dirty(folio, wb);
/* * Pick the slot to use. If there is already a slot for @wb, keep * using it. If not replace the oldest one which isn't being * written out. */ for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { frn = &memcg->cgwb_frn[i]; if (frn->bdi_id == wb->bdi->id && frn->memcg_id == wb->memcg_css->id) break; if (time_before64(frn->at, oldest_at) && atomic_read(&frn->done.cnt) == 1) { oldest = i; oldest_at = frn->at; } }
if (i < MEMCG_CGWB_FRN_CNT) { /* * Re-using an existing one. Update timestamp lazily to * avoid making the cacheline hot. We want them to be * reasonably up-to-date and significantly shorter than * dirty_expire_interval as that's what expires the record. * Use the shorter of 1s and dirty_expire_interval / 8. */ unsigned long update_intv = min_t(unsigned long, HZ, msecs_to_jiffies(dirty_expire_interval * 10) / 8);
if (time_before64(frn->at, now - update_intv)) frn->at = now; } else if (oldest >= 0) { /* replace the oldest free one */ frn = &memcg->cgwb_frn[oldest]; frn->bdi_id = wb->bdi->id; frn->memcg_id = wb->memcg_css->id; frn->at = now; }}
/* issue foreign writeback flushes for recorded foreign dirtying events */void mem_cgroup_flush_foreign(struct bdi_writeback *wb){ struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10); u64 now = get_jiffies_64(); int i;
for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) { struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
/* * If the record is older than dirty_expire_interval, * writeback on it has already started. No need to kick it * off again. Also, don't start a new one if there's * already one in flight. */ if (time_after64(frn->at, now - intv) && atomic_read(&frn->done.cnt) == 1) { frn->at = 0; trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id); cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, WB_REASON_FOREIGN_FLUSH, &frn->done); } }}
#else /* CONFIG_CGROUP_WRITEBACK */
static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp){ return 0;}
static void memcg_wb_domain_exit(struct mem_cgroup *memcg){}
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg){}
#endif /* CONFIG_CGROUP_WRITEBACK */
/* * Private memory cgroup IDR * * Swap-out records and page cache shadow entries need to store memcg * references in constrained space, so we maintain an ID space that is * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of * memory-controlled cgroups to 64k. * * However, there usually are many references to the offline CSS after * the cgroup has been destroyed, such as page cache or reclaimable * slab objects, that don't need to hang on to the ID. We want to keep * those dead CSS from occupying IDs, or we might quickly exhaust the * relatively small ID space and prevent the creation of new cgroups * even when there are much fewer than 64k cgroups - possibly none. * * Maintain a private 16-bit ID space for memcg, and allow the ID to * be freed and recycled when it's no longer needed, which is usually * when the CSS is offlined. * * The only exception to that are records of swapped out tmpfs/shmem * pages that need to be attributed to live ancestors on swapin. But * those references are manageable from userspace. */
#define MEM_CGROUP_ID_MAX ((1UL << MEM_CGROUP_ID_SHIFT) - 1)static DEFINE_XARRAY_ALLOC1(mem_cgroup_private_ids);
static inline struct obj_cgroup *obj_cgroup_from_private_id(unsigned short id){ lockdep_assert_once(rcu_read_lock_held());
return xa_load(&mem_cgroup_private_ids, id);}
static void mem_cgroup_private_id_remove(struct mem_cgroup *memcg){ if (memcg->private_id > 0) { xa_erase(&mem_cgroup_private_ids, memcg->private_id); memcg->private_id = 0; }}
static void __mem_cgroup_private_id_put(struct obj_cgroup *objcg, unsigned short id, unsigned int n){ struct obj_cgroup *objcg_free;
if (refcount_sub_and_test(n, &objcg->private_id_ref)) { objcg_free = xa_erase(&mem_cgroup_private_ids, id); VM_WARN_ON(objcg_free != objcg);
/* Memcg ID pins the objcg */ obj_cgroup_put(objcg); }}
static inline void mem_cgroup_private_id_put(unsigned short id, unsigned int n){ struct obj_cgroup *objcg;
lockdep_assert_in_rcu_read_lock();
objcg = obj_cgroup_from_private_id(id); __mem_cgroup_private_id_put(objcg, id, n);}
static void mem_cgroup_private_id_kill(struct mem_cgroup *memcg){ __mem_cgroup_private_id_put(memcg->private_id_objcg, memcg->private_id, 1);}
unsigned short mem_cgroup_private_id_get(struct mem_cgroup *memcg, unsigned int n){ struct obj_cgroup *objcg;
lockdep_assert_once(rcu_read_lock_held());
objcg = memcg->private_id_objcg;
while (!refcount_add_not_zero(n, &objcg->private_id_ref)) { /* * The root cgroup cannot be destroyed, so it's refcount must * always be >= 1. */ if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) { VM_BUG_ON(1); break; } memcg = parent_mem_cgroup(memcg); objcg = memcg->private_id_objcg; }
return mem_cgroup_private_id(memcg);}
/** * mem_cgroup_from_private_id - look up a memcg from a memcg id * @id: the memcg id to look up * * Caller must hold rcu_read_lock(). * * @return: the memcg, or NULL if the memcg referred to is already dead. */struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id){ struct obj_cgroup *objcg; struct mem_cgroup *memcg;
WARN_ON_ONCE(!rcu_read_lock_held());
objcg = obj_cgroup_from_private_id(id); if (!objcg) return NULL;
memcg = obj_cgroup_memcg(objcg); if (mem_cgroup_private_id(memcg) != id) return NULL;
return memcg;}
struct mem_cgroup *mem_cgroup_get_from_id(u64 id){ struct cgroup *cgrp; struct cgroup_subsys_state *css; struct mem_cgroup *memcg = NULL;
cgrp = cgroup_get_from_id(id); if (IS_ERR(cgrp)) return NULL;
css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); if (css) memcg = container_of(css, struct mem_cgroup, css);
cgroup_put(cgrp);
return memcg;}
static void free_mem_cgroup_per_node_info(struct mem_cgroup_per_node *pn){ if (!pn) return;
free_percpu(pn->lruvec_stats_percpu); kfree(pn->lruvec_stats); kfree(pn);}
static bool alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node){ struct mem_cgroup_per_node *pn;
pn = kmem_cache_alloc_node(memcg_pn_cachep, GFP_KERNEL | __GFP_ZERO, node); if (!pn) return false;
pn->lruvec_stats = kzalloc_node(sizeof(struct lruvec_stats), GFP_KERNEL_ACCOUNT, node); if (!pn->lruvec_stats) goto fail;
pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu, GFP_KERNEL_ACCOUNT); if (!pn->lruvec_stats_percpu) goto fail;
INIT_LIST_HEAD(&pn->objcg_list);
lruvec_init(&pn->lruvec); pn->memcg = memcg;
memcg->nodeinfo[node] = pn; return true;fail: free_mem_cgroup_per_node_info(pn); return false;}
static void __mem_cgroup_free(struct mem_cgroup *memcg){ int node;
for_each_node(node) { struct mem_cgroup_per_node *pn = memcg->nodeinfo[node]; if (!pn) continue;
obj_cgroup_put(pn->orig_objcg); free_mem_cgroup_per_node_info(pn); } memcg1_free_events(memcg); kfree(memcg->vmstats); free_percpu(memcg->vmstats_percpu); kfree(memcg);}
static void mem_cgroup_free(struct mem_cgroup *memcg){ lru_gen_exit_memcg(memcg); memcg_wb_domain_exit(memcg); __mem_cgroup_free(memcg);}
static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent){ struct memcg_vmstats_percpu *statc; struct memcg_vmstats_percpu __percpu *pstatc_pcpu; struct mem_cgroup *memcg; int node, cpu; int __maybe_unused i; long error;
memcg = kmem_cache_zalloc(memcg_cachep, GFP_KERNEL); if (!memcg) return ERR_PTR(-ENOMEM);
error = xa_alloc(&mem_cgroup_private_ids, &memcg->private_id, NULL, XA_LIMIT(1, MEM_CGROUP_ID_MAX), GFP_KERNEL); if (error) goto fail; error = -ENOMEM;
memcg->vmstats = kzalloc_obj(struct memcg_vmstats, GFP_KERNEL_ACCOUNT); if (!memcg->vmstats) goto fail;
memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, GFP_KERNEL_ACCOUNT); if (!memcg->vmstats_percpu) goto fail;
if (!memcg1_alloc_events(memcg)) goto fail;
pstatc_pcpu = parent ? parent->vmstats_percpu : NULL; for_each_possible_cpu(cpu) { statc = per_cpu_ptr(memcg->vmstats_percpu, cpu); statc->parent_pcpu = pstatc_pcpu; statc->vmstats = memcg->vmstats; }
for_each_node(node) if (!alloc_mem_cgroup_per_node_info(memcg, node)) goto fail;
if (memcg_wb_domain_init(memcg, GFP_KERNEL)) goto fail;
INIT_WORK(&memcg->high_work, high_work_func); init_irq_work(&memcg->high_irq_work, high_irq_work_func); vmpressure_init(&memcg->vmpressure); INIT_LIST_HEAD(&memcg->memory_peaks); INIT_LIST_HEAD(&memcg->swap_peaks); spin_lock_init(&memcg->peaks_lock); memcg->socket_pressure = get_jiffies_64();#if BITS_PER_LONG < 64 seqlock_init(&memcg->socket_pressure_seqlock);#endif memcg1_memcg_init(memcg); memcg->kmemcg_id = -1;#ifdef CONFIG_CGROUP_WRITEBACK INIT_LIST_HEAD(&memcg->cgwb_list); for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) memcg->cgwb_frn[i].done = __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);#endif lru_gen_init_memcg(memcg); return memcg;fail: mem_cgroup_private_id_remove(memcg); __mem_cgroup_free(memcg); return ERR_PTR(error);}
static struct cgroup_subsys_state * __refmem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css){ struct mem_cgroup *parent = mem_cgroup_from_css(parent_css); struct mem_cgroup *memcg, *old_memcg; bool memcg_on_dfl = cgroup_subsys_on_dfl(memory_cgrp_subsys);
old_memcg = set_active_memcg(parent); memcg = mem_cgroup_alloc(parent); set_active_memcg(old_memcg); if (IS_ERR(memcg)) return ERR_CAST(memcg);
page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);#ifdef CONFIG_ZSWAP memcg->zswap_max = PAGE_COUNTER_MAX; WRITE_ONCE(memcg->zswap_writeback, true);#endif page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); if (parent) { page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl); page_counter_init(&memcg->swap, &parent->swap, false);#ifdef CONFIG_MEMCG_V1 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); memcg->memory.track_failcnt = !memcg_on_dfl; memcg->memsw.track_failcnt = !memcg_on_dfl; WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable)); page_counter_init(&memcg->kmem, &parent->kmem, false); page_counter_init(&memcg->tcpmem, &parent->tcpmem, false); memcg->tcpmem.track_failcnt = !memcg_on_dfl;#endif } else { init_memcg_stats(); init_memcg_events(); page_counter_init(&memcg->memory, NULL, true); page_counter_init(&memcg->swap, NULL, false);#ifdef CONFIG_MEMCG_V1 page_counter_init(&memcg->kmem, NULL, false); page_counter_init(&memcg->tcpmem, NULL, false);#endif root_mem_cgroup = memcg; return &memcg->css; }
if (memcg_on_dfl && !cgroup_memory_nosocket) static_branch_inc(&memcg_sockets_enabled_key);
if (!cgroup_memory_nobpf) static_branch_inc(&memcg_bpf_enabled_key);
return &memcg->css;}
static int mem_cgroup_css_online(struct cgroup_subsys_state *css){ struct mem_cgroup *memcg = mem_cgroup_from_css(css); struct obj_cgroup *objcg; int nid;
memcg_online_kmem(memcg);
/* * A memcg must be visible for expand_shrinker_info() * by the time the maps are allocated. So, we allocate maps * here, when mem_cgroup_iter() can't skip it. */ if (alloc_shrinker_info(memcg)) goto offline_kmem;
for_each_node(nid) { objcg = obj_cgroup_alloc(); if (!objcg) goto free_objcg;
if (unlikely(mem_cgroup_is_root(memcg))) objcg->is_root = true;
objcg->memcg = memcg; rcu_assign_pointer(memcg->nodeinfo[nid]->objcg, objcg); obj_cgroup_get(objcg); memcg->nodeinfo[nid]->orig_objcg = objcg; }
if (unlikely(mem_cgroup_is_root(memcg)) && !mem_cgroup_disabled()) queue_delayed_work(system_dfl_wq, &stats_flush_dwork, FLUSH_TIME); lru_gen_online_memcg(memcg);
/* CSS pins memcg ID, memcg ID pins obj cgroup */ memcg->private_id_objcg = objcg; refcount_set(&memcg->private_id_objcg->private_id_ref, 1); obj_cgroup_get(memcg->private_id_objcg);
/* * Ensure mem_cgroup_from_private_id() works once we're fully online. * * We could do this earlier and require callers to filter with * css_tryget_online(). But right now there are no users that * need earlier access, and the workingset code relies on the * cgroup tree linkage (mem_cgroup_get_nr_swap_pages()). So * publish it here at the end of onlining. This matches the * regular ID destruction during offlining. */ xa_store(&mem_cgroup_private_ids, memcg->private_id, memcg->private_id_objcg, GFP_KERNEL);
return 0;free_objcg: for_each_node(nid) { struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
objcg = rcu_replace_pointer(pn->objcg, NULL, true); if (objcg) percpu_ref_kill(&objcg->refcnt);
if (pn->orig_objcg) { obj_cgroup_put(pn->orig_objcg); /* * Reset pn->orig_objcg to NULL to prevent * obj_cgroup_put() from being called again in * __mem_cgroup_free(). */ pn->orig_objcg = NULL; } } free_shrinker_info(memcg);offline_kmem: memcg_offline_kmem(memcg); mem_cgroup_private_id_remove(memcg); return -ENOMEM;}
static void mem_cgroup_css_offline(struct cgroup_subsys_state *css){ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
memcg1_css_offline(memcg);
page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0);
zswap_memcg_offline_cleanup(memcg);
memcg_offline_kmem(memcg); /* * The reparenting of objcg must be after the reparenting of * the list_lru in memcg_offline_kmem(), which ensures that * they will not mistakenly get the parent list_lru. */ memcg_reparent_objcgs(memcg); reparent_shrinker_deferred(memcg); wb_memcg_offline(memcg); lru_gen_offline_memcg(memcg);
drain_all_stock(memcg);
mem_cgroup_private_id_kill(memcg);}
static void mem_cgroup_css_released(struct cgroup_subsys_state *css){ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
invalidate_reclaim_iterators(memcg); lru_gen_release_memcg(memcg);}
static void mem_cgroup_css_free(struct cgroup_subsys_state *css){ struct mem_cgroup *memcg = mem_cgroup_from_css(css); int __maybe_unused i;
#ifdef CONFIG_CGROUP_WRITEBACK for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) wb_wait_for_completion(&memcg->cgwb_frn[i].done);#endif if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket) static_branch_dec(&memcg_sockets_enabled_key);
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg1_tcpmem_active(memcg)) static_branch_dec(&memcg_sockets_enabled_key);
if (!cgroup_memory_nobpf) static_branch_dec(&memcg_bpf_enabled_key);
vmpressure_cleanup(&memcg->vmpressure); irq_work_sync(&memcg->high_irq_work); cancel_work_sync(&memcg->high_work); free_shrinker_info(memcg); mem_cgroup_free(memcg);}
/** * mem_cgroup_css_reset - reset the states of a mem_cgroup * @css: the target css * * Reset the states of the mem_cgroup associated with @css. This is * invoked when the userland requests disabling on the default hierarchy * but the memcg is pinned through dependency. The memcg should stop * applying policies and should revert to the vanilla state as it may be * made visible again. * * The current implementation only resets the essential configurations. * This needs to be expanded to cover all the visible parts. */static void mem_cgroup_css_reset(struct cgroup_subsys_state *css){ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX); WRITE_ONCE(memcg->oom_group, false);#ifdef CONFIG_ZSWAP WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX); WRITE_ONCE(memcg->zswap_writeback, true);#endif#ifdef CONFIG_MEMCG_V1 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);#endif page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0); page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX); page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); memcg_wb_domain_size_changed(memcg);}
struct aggregate_control { /* pointer to the aggregated (CPU and subtree aggregated) counters */ long *aggregate; /* * pointer to the non-hierarchical (CPU aggregated) counters or NULL to * skip updating them (see memcg_state_local_array()) */ long *local; /* pointer to the pending child counters during tree propagation */ long *pending; /* pointer to the parent's pending counters, could be NULL */ long *ppending; /* pointer to the percpu counters to be aggregated */ long *cstat; /* pointer to the percpu counters of the last aggregation*/ long *cstat_prev; /* size of the above counters */ int size;};
static void mem_cgroup_stat_aggregate(struct aggregate_control *ac){ int i; long delta, delta_cpu, v;
for (i = 0; i < ac->size; i++) { /* * Collect the aggregated propagation counts of groups * below us. We're in a per-cpu loop here and this is * a global counter, so the first cycle will get them. */ delta = ac->pending[i]; if (delta) ac->pending[i] = 0;
/* Add CPU changes on this level since the last flush */ delta_cpu = 0; v = READ_ONCE(ac->cstat[i]); if (v != ac->cstat_prev[i]) { delta_cpu = v - ac->cstat_prev[i]; delta += delta_cpu; ac->cstat_prev[i] = v; }
/* Aggregate counts on this level and propagate upwards */ if (delta_cpu && ac->local) ac->local[i] += delta_cpu;
if (delta) { ac->aggregate[i] += delta; if (ac->ppending) ac->ppending[i] += delta; } }}
#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMICstatic void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent){ long *state_local = memcg_state_local_array(memcg); int nid;
if (atomic_read(&memcg->kmem_stat)) { int kmem = atomic_xchg(&memcg->kmem_stat, 0); int index = memcg_stats_index(MEMCG_KMEM);
memcg->vmstats->state[index] += kmem; if (state_local) state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; }
for_each_node_state(nid, N_MEMORY) { struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; struct lruvec_stats *lstats = pn->lruvec_stats; struct lruvec_stats *plstats = NULL;
if (parent) plstats = parent->nodeinfo[nid]->lruvec_stats;
if (atomic_read(&pn->slab_reclaimable)) { int slab = atomic_xchg(&pn->slab_reclaimable, 0); int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B);
lstats->state[index] += slab; lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; if (state_local) state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } if (atomic_read(&pn->slab_unreclaimable)) { int slab = atomic_xchg(&pn->slab_unreclaimable, 0); int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B);
lstats->state[index] += slab; lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; if (state_local) state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } }}#elsestatic void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent){}#endif
static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu){ struct mem_cgroup *memcg = mem_cgroup_from_css(css); struct mem_cgroup *parent = parent_mem_cgroup(memcg); struct memcg_vmstats_percpu *statc; struct aggregate_control ac; int nid;
flush_nmi_stats(memcg, parent);
statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
ac = (struct aggregate_control) { .aggregate = memcg->vmstats->state, .local = memcg_state_local_array(memcg), .pending = memcg->vmstats->state_pending, .ppending = parent ? parent->vmstats->state_pending : NULL, .cstat = statc->state, .cstat_prev = statc->state_prev, .size = MEMCG_VMSTAT_SIZE, }; mem_cgroup_stat_aggregate(&ac);
ac = (struct aggregate_control) { .aggregate = memcg->vmstats->events, .local = memcg_events_local_array(memcg), .pending = memcg->vmstats->events_pending, .ppending = parent ? parent->vmstats->events_pending : NULL, .cstat = statc->events, .cstat_prev = statc->events_prev, .size = NR_MEMCG_EVENTS, }; mem_cgroup_stat_aggregate(&ac);
for_each_node_state(nid, N_MEMORY) { struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; struct lruvec_stats *lstats = pn->lruvec_stats; struct lruvec_stats *plstats = NULL; struct lruvec_stats_percpu *lstatc;
if (parent) plstats = parent->nodeinfo[nid]->lruvec_stats;
lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu);
ac = (struct aggregate_control) { .aggregate = lstats->state, .local = lstats->state_local, .pending = lstats->state_pending, .ppending = plstats ? plstats->state_pending : NULL, .cstat = lstatc->state, .cstat_prev = lstatc->state_prev, .size = NR_MEMCG_NODE_STAT_ITEMS, }; mem_cgroup_stat_aggregate(&ac);
} WRITE_ONCE(statc->stats_updates, 0); /* We are in a per-cpu loop here, only do the atomic write once */ if (atomic_long_read(&memcg->vmstats->stats_updates)) atomic_long_set(&memcg->vmstats->stats_updates, 0);}
static void mem_cgroup_fork(struct task_struct *task){ /* * Set the update flag to cause task->objcg to be initialized lazily * on the first allocation. It can be done without any synchronization * because it's always performed on the current task, so does * current_objcg_update(). */ task->objcg = (struct obj_cgroup *)CURRENT_OBJCG_UPDATE_FLAG;}
static void mem_cgroup_exit(struct task_struct *task){ struct obj_cgroup *objcg = task->objcg;
objcg = (struct obj_cgroup *) ((unsigned long)objcg & ~CURRENT_OBJCG_UPDATE_FLAG); obj_cgroup_put(objcg);
/* * Some kernel allocations can happen after this point, * but let's ignore them. It can be done without any synchronization * because it's always performed on the current task, so does * current_objcg_update(). */ task->objcg = NULL;}
#ifdef CONFIG_LRU_GENstatic void mem_cgroup_lru_gen_attach(struct cgroup_taskset *tset){ struct task_struct *task; struct cgroup_subsys_state *css;
/* find the first leader if there is any */ cgroup_taskset_for_each_leader(task, css, tset) break;
if (!task) return;
task_lock(task); if (task->mm && READ_ONCE(task->mm->owner) == task) lru_gen_migrate_mm(task->mm); task_unlock(task);}#elsestatic void mem_cgroup_lru_gen_attach(struct cgroup_taskset *tset) {}#endif /* CONFIG_LRU_GEN */
static void mem_cgroup_kmem_attach(struct cgroup_taskset *tset){ struct task_struct *task; struct cgroup_subsys_state *css;
cgroup_taskset_for_each(task, css, tset) { /* atomically set the update bit */ set_bit(CURRENT_OBJCG_UPDATE_BIT, (unsigned long *)&task->objcg); }}
static void mem_cgroup_attach(struct cgroup_taskset *tset){ mem_cgroup_lru_gen_attach(tset); mem_cgroup_kmem_attach(tset);}
static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value){ if (value == PAGE_COUNTER_MAX) seq_puts(m, "max\n"); else seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
return 0;}
static u64 memory_current_read(struct cgroup_subsys_state *css, struct cftype *cft){ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;}
#define OFP_PEAK_UNSET (((-1UL)))
static int peak_show(struct seq_file *sf, void *v, struct page_counter *pc){ struct cgroup_of_peak *ofp = of_peak(sf->private); u64 fd_peak = READ_ONCE(ofp->value), peak;
/* User wants global or local peak? */ if (fd_peak == OFP_PEAK_UNSET) peak = pc->watermark; else peak = max(fd_peak, READ_ONCE(pc->local_watermark));
seq_printf(sf, "%llu\n", peak * PAGE_SIZE); return 0;}
static int memory_peak_show(struct seq_file *sf, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
guard(spinlock)(&memcg->peaks_lock); return peak_show(sf, v, &memcg->memory);}
static int peak_open(struct kernfs_open_file *of){ struct cgroup_of_peak *ofp = of_peak(of);
ofp->value = OFP_PEAK_UNSET; return 0;}
static void peak_release(struct kernfs_open_file *of){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); struct cgroup_of_peak *ofp = of_peak(of);
if (ofp->value == OFP_PEAK_UNSET) { /* fast path (no writes on this fd) */ return; } spin_lock(&memcg->peaks_lock); list_del(&ofp->list); spin_unlock(&memcg->peaks_lock);}
static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off, struct page_counter *pc, struct list_head *watchers){ unsigned long usage, old_watermark; struct cgroup_of_peak *peer_ctx; struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); struct cgroup_of_peak *ofp = of_peak(of);
spin_lock(&memcg->peaks_lock);
usage = page_counter_read(pc); old_watermark = READ_ONCE(pc->local_watermark); WRITE_ONCE(pc->local_watermark, usage);
list_for_each_entry(peer_ctx, watchers, list) if (peer_ctx != ofp && old_watermark > peer_ctx->value) WRITE_ONCE(peer_ctx->value, old_watermark);
/* initial write, register watcher */ if (ofp->value == OFP_PEAK_UNSET) list_add(&ofp->list, watchers);
WRITE_ONCE(ofp->value, usage); spin_unlock(&memcg->peaks_lock);
return nbytes;}
static ssize_t memory_peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
return peak_write(of, buf, nbytes, off, &memcg->memory, &memcg->memory_peaks);}
#undef OFP_PEAK_UNSET
static int memory_min_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->memory.min));}
static ssize_t memory_min_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned long min; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &min); if (err) return err;
page_counter_set_min(&memcg->memory, min);
return nbytes;}
static int memory_low_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->memory.low));}
static ssize_t memory_low_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned long low; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &low); if (err) return err;
page_counter_set_low(&memcg->memory, low);
return nbytes;}
static int memory_high_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->memory.high));}
static ssize_t memory_high_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned int nr_retries = MAX_RECLAIM_RETRIES; bool drained = false; unsigned long high; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &high); if (err) return err;
page_counter_set_high(&memcg->memory, high);
if (of->file->f_flags & O_NONBLOCK) goto out;
for (;;) { unsigned long nr_pages = page_counter_read(&memcg->memory); unsigned long reclaimed;
if (high != READ_ONCE(memcg->memory.high)) break;
if (nr_pages <= high) break;
if (signal_pending(current)) break;
/* cgroup_rmdir() waits for us with cgroup_mutex held. */ if (memcg_is_dying(memcg)) break;
if (!drained) { drain_all_stock(memcg); drained = true; continue; }
reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high, GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL);
if (!reclaimed && !nr_retries--) break; }out: memcg_wb_domain_size_changed(memcg); return nbytes;}
static int memory_max_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->memory.max));}
static ssize_t memory_max_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned int nr_reclaims = MAX_RECLAIM_RETRIES; bool drained = false; unsigned long max; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &max); if (err) return err;
xchg(&memcg->memory.max, max);
if (of->file->f_flags & O_NONBLOCK) goto out;
for (;;) { unsigned long nr_pages = page_counter_read(&memcg->memory);
if (max != READ_ONCE(memcg->memory.max)) break;
if (nr_pages <= max) break;
if (signal_pending(current)) break;
/* cgroup_rmdir() waits for us with cgroup_mutex held. */ if (memcg_is_dying(memcg)) break;
if (!drained) { drain_all_stock(memcg); drained = true; continue; }
if (nr_reclaims) { if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max, GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL)) nr_reclaims--; continue; }
memcg_memory_event(memcg, MEMCG_OOM); if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) break; cond_resched(); }out: memcg_wb_domain_size_changed(memcg); return nbytes;}
/* * Note: don't forget to update the 'samples/cgroup/memcg_event_listener' * if any new events become available. */static void __memory_events_show(struct seq_file *m, atomic_long_t *events){ seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW])); seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH])); seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX])); seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM])); seq_printf(m, "oom_kill %lu\n", atomic_long_read(&events[MEMCG_OOM_KILL])); seq_printf(m, "oom_group_kill %lu\n", atomic_long_read(&events[MEMCG_OOM_GROUP_KILL])); seq_printf(m, "sock_throttled %lu\n", atomic_long_read(&events[MEMCG_SOCK_THROTTLED]));}
static int memory_events_show(struct seq_file *m, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
__memory_events_show(m, memcg->memory_events); return 0;}
static int memory_events_local_show(struct seq_file *m, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
__memory_events_show(m, memcg->memory_events_local); return 0;}
int memory_stat_show(struct seq_file *m, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_seq(m); char *buf = kmalloc(SEQ_BUF_SIZE, GFP_KERNEL); struct seq_buf s;
if (!buf) return -ENOMEM; seq_buf_init(&s, buf, SEQ_BUF_SIZE); memory_stat_format(memcg, &s); seq_puts(m, buf); kfree(buf); return 0;}
#ifdef CONFIG_NUMAstatic inline unsigned long lruvec_page_state_output(struct lruvec *lruvec, int item){ return lruvec_page_state(lruvec, item) * memcg_page_state_output_unit(item);}
static int memory_numa_stat_show(struct seq_file *m, void *v){ int i; struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
mem_cgroup_flush_stats(memcg);
for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { int nid;
if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS) continue;
seq_printf(m, "%s", memory_stats[i].name); for_each_node_state(nid, N_MEMORY) { u64 size; struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); size = lruvec_page_state_output(lruvec, memory_stats[i].idx); seq_printf(m, " N%d=%llu", nid, size); } seq_putc(m, '\n'); }
return 0;}#endif
static int memory_oom_group_show(struct seq_file *m, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group));
return 0;}
static ssize_t memory_oom_group_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); int ret, oom_group;
buf = strstrip(buf); if (!buf) return -EINVAL;
ret = kstrtoint(buf, 0, &oom_group); if (ret) return ret;
if (oom_group != 0 && oom_group != 1) return -EINVAL;
WRITE_ONCE(memcg->oom_group, oom_group);
return nbytes;}
static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); int ret;
ret = user_proactive_reclaim(buf, memcg, NULL); if (ret) return ret;
return nbytes;}
static struct cftype memory_files[] = { { .name = "current", .flags = CFTYPE_NOT_ON_ROOT, .read_u64 = memory_current_read, }, { .name = "peak", .flags = CFTYPE_NOT_ON_ROOT, .open = peak_open, .release = peak_release, .seq_show = memory_peak_show, .write = memory_peak_write, }, { .name = "min", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = memory_min_show, .write = memory_min_write, }, { .name = "low", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = memory_low_show, .write = memory_low_write, }, { .name = "high", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = memory_high_show, .write = memory_high_write, }, { .name = "max", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = memory_max_show, .write = memory_max_write, }, { .name = "events", .flags = CFTYPE_NOT_ON_ROOT, .file_offset = offsetof(struct mem_cgroup, events_file), .seq_show = memory_events_show, }, { .name = "events.local", .flags = CFTYPE_NOT_ON_ROOT, .file_offset = offsetof(struct mem_cgroup, events_local_file), .seq_show = memory_events_local_show, }, { .name = "stat", .seq_show = memory_stat_show, },#ifdef CONFIG_NUMA { .name = "numa_stat", .seq_show = memory_numa_stat_show, },#endif { .name = "oom.group", .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE, .seq_show = memory_oom_group_show, .write = memory_oom_group_write, }, { .name = "reclaim", .flags = CFTYPE_NS_DELEGATABLE, .write = memory_reclaim, }, { } /* terminate */};
struct cgroup_subsys memory_cgrp_subsys = { .css_alloc = mem_cgroup_css_alloc, .css_online = mem_cgroup_css_online, .css_offline = mem_cgroup_css_offline, .css_released = mem_cgroup_css_released, .css_free = mem_cgroup_css_free, .css_reset = mem_cgroup_css_reset, .css_rstat_flush = mem_cgroup_css_rstat_flush, .attach = mem_cgroup_attach, .fork = mem_cgroup_fork, .exit = mem_cgroup_exit, .dfl_cftypes = memory_files,#ifdef CONFIG_MEMCG_V1 .legacy_cftypes = mem_cgroup_legacy_files,#endif .early_init = 0,};
/** * mem_cgroup_calculate_protection - check if memory consumption is in the normal range * @root: the top ancestor of the sub-tree being checked * @memcg: the memory cgroup to check * * WARNING: This function is not stateless! It can only be used as part * of a top-down tree iteration, not for isolated queries. */void mem_cgroup_calculate_protection(struct mem_cgroup *root, struct mem_cgroup *memcg){ bool recursive_protection = cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT;
if (mem_cgroup_disabled()) return;
if (!root) root = root_mem_cgroup;
page_counter_calculate_protection(&root->memory, &memcg->memory, recursive_protection);}
#ifdef CONFIG_LRU_GEN/** * mem_cgroup_calculate_protection_path - compute protection along a path * @root: the top ancestor of the sub-tree being checked (NULL for root_mem_cgroup) * @memcg: the target memory cgroup * * Walk the ancestor path from @root down to @memcg and compute the effective * protection at each level. This is safe for isolated queries because it * ensures parents are computed before children. */void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, struct mem_cgroup *memcg){ bool recursive_protection = cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT; struct cgroup *cg; int root_level, i;
if (mem_cgroup_disabled()) return;
if (!root) root = root_mem_cgroup;
if (memcg == root) return;
root_level = root->css.cgroup->level; cg = memcg->css.cgroup;
rcu_read_lock(); for (i = root_level + 1; i <= cg->level; i++) { struct mem_cgroup *cur;
cur = mem_cgroup_from_css(cgroup_css(cg->ancestors[i], &memory_cgrp_subsys)); if (cur) page_counter_calculate_protection(&root->memory, &cur->memory, recursive_protection); } rcu_read_unlock();}#endif /* CONFIG_LRU_GEN */
static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg, gfp_t gfp){ int ret = 0; struct obj_cgroup *objcg;
objcg = get_obj_cgroup_from_memcg(memcg, folio_nid(folio)); /* Do not account at the root objcg level. */ if (!obj_cgroup_is_root(objcg)) ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio)); if (ret) { obj_cgroup_put(objcg); return ret; } commit_charge(folio, objcg); memcg1_commit_charge(folio, memcg);
return ret;}
int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp){ struct mem_cgroup *memcg; int ret;
memcg = get_mem_cgroup_from_mm(mm); ret = charge_memcg(folio, memcg, gfp); css_put(&memcg->css);
return ret;}
/** * mem_cgroup_charge_hugetlb - charge the memcg for a hugetlb folio * @folio: folio being charged * @mm: mm context of the allocation target * @gfp: reclaim mode * * This function is called when allocating a huge page folio, after the page has * already been obtained and charged to the appropriate hugetlb cgroup * controller (if it is enabled). * * Returns ENOMEM if the memcg is already full. * Returns 0 if either the charge was successful, or if we skip the charging. */int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, gfp_t gfp){ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); int ret = 0;
/* * Even memcg does not account for hugetlb, we still want to update * system-level stats via lruvec_stat_mod_folio. Return 0, and skip * charging the memcg. */ if (mem_cgroup_disabled() || !memcg_accounts_hugetlb() || !memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) goto out;
if (charge_memcg(folio, memcg, gfp)) ret = -ENOMEM;
out: mem_cgroup_put(memcg); return ret;}
/** * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin. * @folio: the folio to charge * @id: memory cgroup id * @mm: mm context of the victim * @gfp: reclaim mode * * This function charges a folio allocated for swapin. Please call this before * adding the folio to the swapcache. * * Returns 0 on success. Otherwise, an error code is returned. */int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp){ struct mem_cgroup *memcg; int ret;
if (mem_cgroup_disabled()) return 0;
rcu_read_lock(); memcg = mem_cgroup_from_private_id(id); if (!memcg || !css_tryget_online(&memcg->css)) memcg = get_mem_cgroup_from_mm(mm); rcu_read_unlock();
ret = charge_memcg(folio, memcg, gfp);
css_put(&memcg->css); return ret;}
struct uncharge_gather { struct obj_cgroup *objcg; unsigned long nr_memory; unsigned long pgpgout; unsigned long nr_kmem;};
static inline void uncharge_gather_clear(struct uncharge_gather *ug){ memset(ug, 0, sizeof(*ug));}
static void uncharge_batch(const struct uncharge_gather *ug){ struct mem_cgroup *memcg;
rcu_read_lock(); memcg = obj_cgroup_memcg(ug->objcg); if (ug->nr_memory) { memcg_uncharge(memcg, ug->nr_memory); if (ug->nr_kmem) { mod_memcg_state(memcg, MEMCG_KMEM, -ug->nr_kmem); memcg1_account_kmem(memcg, -ug->nr_kmem); } memcg1_oom_recover(memcg); }
memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory); rcu_read_unlock();
/* drop reference from uncharge_folio */ obj_cgroup_put(ug->objcg);}
static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug){ long nr_pages; struct obj_cgroup *objcg;
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
/* * Nobody should be changing or seriously looking at * folio objcg at this point, we have fully exclusive * access to the folio. */ objcg = folio_objcg(folio); if (!objcg) return;
if (ug->objcg != objcg) { if (ug->objcg) { uncharge_batch(ug); uncharge_gather_clear(ug); } ug->objcg = objcg;
/* pairs with obj_cgroup_put in uncharge_batch */ obj_cgroup_get(objcg); }
nr_pages = folio_nr_pages(folio);
if (folio_memcg_kmem(folio)) { ug->nr_memory += nr_pages; ug->nr_kmem += nr_pages; } else { /* LRU pages aren't accounted at the root level */ if (!obj_cgroup_is_root(objcg)) ug->nr_memory += nr_pages; ug->pgpgout++;
WARN_ON_ONCE(folio_unqueue_deferred_split(folio)); }
folio->memcg_data = 0; obj_cgroup_put(objcg);}
void __mem_cgroup_uncharge(struct folio *folio){ struct uncharge_gather ug;
/* Don't touch folio->lru of any random page, pre-check: */ if (!folio_memcg_charged(folio)) return;
uncharge_gather_clear(&ug); uncharge_folio(folio, &ug); uncharge_batch(&ug);}
void __mem_cgroup_uncharge_folios(struct folio_batch *folios){ struct uncharge_gather ug; unsigned int i;
uncharge_gather_clear(&ug); for (i = 0; i < folios->nr; i++) uncharge_folio(folios->folios[i], &ug); if (ug.objcg) uncharge_batch(&ug);}
/* * An LRU folio must hold the objcg belonging to its own node. * * memcg_reparent_objcgs() reparents a dying cgroup one node at a time: the * folios on that node's LRU lists move to the parent and that node's objcg is * redirected to the parent, atomically under the node's lru_lock. * folio_lruvec_lock() relies on this to provide a stable folio<->lruvec * binding. If a folio holds another node's objcg, its list membership and its * lruvec resolution change in separate lock sections, and an LRU operation in * between can re-add the folio to, and strand it on, the LRU list of a dead * memcg. * * So when migration transfers the memcg state to a folio on another node, * re-derive the objcg for the destination node. If the memcg is dying and the * destination node has already been reparented, the lookup walks up to the * nearest live ancestor - which is also where that node's LRU lists went. * * Returns the objcg to commit to @new, with a reference for the caller. */static struct obj_cgroup *get_migration_objcg(struct folio *old, struct folio *new){ struct obj_cgroup *old_objcg, *new_objcg; int new_nid = folio_nid(new);
old_objcg = get_obj_cgroup_from_folio(old);
if (folio_nid(old) == new_nid) return old_objcg;
rcu_read_lock(); new_objcg = __get_obj_cgroup_from_memcg(obj_cgroup_memcg(old_objcg), new_nid); rcu_read_unlock();
obj_cgroup_put(old_objcg);
return new_objcg;}
/** * mem_cgroup_replace_folio - Charge a folio's replacement. * @old: Currently circulating folio. * @new: Replacement folio. * * Charge @new as a replacement folio for @old. @old will * be uncharged upon free. * * Both folios must be locked, @new->mapping must be set up. */void mem_cgroup_replace_folio(struct folio *old, struct folio *new){ struct mem_cgroup *memcg; struct obj_cgroup *objcg; long nr_pages = folio_nr_pages(new);
VM_BUG_ON_FOLIO(!folio_test_locked(old), old); VM_BUG_ON_FOLIO(!folio_test_locked(new), new); VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new); VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new);
if (mem_cgroup_disabled()) return;
/* Page cache replacement: new folio already charged? */ if (folio_memcg_charged(new)) return;
VM_WARN_ON_ONCE_FOLIO(!folio_objcg(old), old); if (!folio_objcg(old)) return;
objcg = get_migration_objcg(old, new);
rcu_read_lock(); memcg = obj_cgroup_memcg(objcg);
/* * Force-charge the new page. The old one will be freed soon. * * The rootness of the committed objcg decides whether the final * uncharge of @new goes through the page counters (see * uncharge_folio()); charge them only if the uncharge will. */ if (!obj_cgroup_is_root(objcg)) { page_counter_charge(&memcg->memory, nr_pages); if (do_memsw_account()) page_counter_charge(&memcg->memsw, nr_pages); }
commit_charge(new, objcg); memcg1_commit_charge(new, memcg); rcu_read_unlock();}
/** * mem_cgroup_migrate - Transfer the memcg data from the old to the new folio. * @old: Currently circulating folio. * @new: Replacement folio. * * Transfer the memcg data from the old folio to the new folio for migration. * The old folio's data info will be cleared. The memory counters remain * unchanged, unless the charge moves out of a fully reparented ancestry * and has to be settled (see below). * * Both folios must be locked, @new->mapping must be set up. */void mem_cgroup_migrate(struct folio *old, struct folio *new){ struct obj_cgroup *objcg, *new_objcg;
VM_BUG_ON_FOLIO(!folio_test_locked(old), old); VM_BUG_ON_FOLIO(!folio_test_locked(new), new); VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new); VM_BUG_ON_FOLIO(folio_nr_pages(old) != folio_nr_pages(new), new); VM_BUG_ON_FOLIO(folio_test_lru(old), old);
if (mem_cgroup_disabled()) return;
objcg = folio_objcg(old); /* * Note that it is normal to see !objcg for a hugetlb folio. * For e.g, it could have been allocated when memory_hugetlb_accounting * was not selected. */ VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(old) && !objcg, old); if (!objcg) return;
new_objcg = get_migration_objcg(old, new);
/* * @old was charged through a non-root objcg, so its charge is in the * page counters. If the re-derivation walked up to the root objcg - * @old's entire ancestry is dying and already reparented - the final * uncharge of @new will skip the page counters (see uncharge_folio()). * Settle them now: this is @old's eventual uncharge, moved up to the * point where its charge record ends. */ if (obj_cgroup_is_root(new_objcg) && !obj_cgroup_is_root(objcg)) { rcu_read_lock(); memcg_uncharge(obj_cgroup_memcg(objcg), folio_nr_pages(old)); rcu_read_unlock(); }
commit_charge(new, new_objcg);
/* Warning should never happen, so don't worry about refcount non-0 */ WARN_ON_ONCE(folio_unqueue_deferred_split(old)); old->memcg_data = 0;
/* @new holds its own reference now, drop @old's */ obj_cgroup_put(objcg);}
DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);EXPORT_SYMBOL(memcg_sockets_enabled_key);
void mem_cgroup_sk_alloc(struct sock *sk){ struct mem_cgroup *memcg;
if (!mem_cgroup_sockets_enabled) return;
/* Do not associate the sock with unrelated interrupted task's memcg. */ if (!in_task()) return;
rcu_read_lock(); memcg = mem_cgroup_from_task(current); if (mem_cgroup_is_root(memcg)) goto out; if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg1_tcpmem_active(memcg)) goto out; if (css_tryget(&memcg->css)) sk->sk_memcg = memcg;out: rcu_read_unlock();}
void mem_cgroup_sk_free(struct sock *sk){ struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
if (memcg) css_put(&memcg->css);}
void mem_cgroup_sk_inherit(const struct sock *sk, struct sock *newsk){ struct mem_cgroup *memcg;
if (sk->sk_memcg == newsk->sk_memcg) return;
mem_cgroup_sk_free(newsk);
memcg = mem_cgroup_from_sk(sk); if (memcg) css_get(&memcg->css);
newsk->sk_memcg = sk->sk_memcg;}
/** * mem_cgroup_sk_charge - charge socket memory * @sk: socket in memcg to charge * @nr_pages: number of pages to charge * @gfp_mask: reclaim mode * * Charges @nr_pages to @memcg. Returns %true if the charge fit within * @memcg's configured limit, %false if it doesn't. */bool mem_cgroup_sk_charge(const struct sock *sk, unsigned int nr_pages, gfp_t gfp_mask){ struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return memcg1_charge_skmem(memcg, nr_pages, gfp_mask);
if (try_charge_memcg(memcg, gfp_mask, nr_pages) == 0) { mod_memcg_state(memcg, MEMCG_SOCK, nr_pages); return true; }
return false;}
/** * mem_cgroup_sk_uncharge - uncharge socket memory * @sk: socket in memcg to uncharge * @nr_pages: number of pages to uncharge */void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages){ struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) { memcg1_uncharge_skmem(memcg, nr_pages); return; }
mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
refill_stock(memcg, nr_pages);}
void mem_cgroup_flush_workqueue(void){ flush_workqueue(memcg_wq);}
static int __init cgroup_memory(char *s){ char *token;
while ((token = strsep(&s, ",")) != NULL) { if (!*token) continue; if (!strcmp(token, "nosocket")) cgroup_memory_nosocket = true; if (!strcmp(token, "nokmem")) cgroup_memory_nokmem = true; if (!strcmp(token, "nobpf")) cgroup_memory_nobpf = true; } return 1;}__setup("cgroup.memory=", cgroup_memory);
/* * Memory controller init before cgroup_init() initialize root_mem_cgroup. * * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this * context because of lock dependencies (cgroup_lock -> cpu hotplug) but * basically everything that doesn't depend on a specific mem_cgroup structure * should be initialized from here. *//* * Fields are grouped by access pattern. Putting a field in the wrong group * breaks the build here. */static void __init memcg_struct_check(void){ CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, socket_pressure);#if BITS_PER_LONG < 64 CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, socket_pressure_seqlock);#endif CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, memory_events); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, memory_events_local); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, vmpressure);#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, kmem_stat);#endif CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, high_irq_work); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot, high_work);
CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, memory_peaks); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, swap_peaks); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, peaks_lock); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, events_file); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, events_local_file); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, swap_events_file);#ifdef CONFIG_CGROUP_WRITEBACK CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, cgwb_list); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, cgwb_domain); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, cgwb_frn);#endif#ifdef CONFIG_LRU_GEN_WALKS_MMU CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold, mm_list);#endif
CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, vmstats_percpu); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, vmstats);#ifdef CONFIG_ZSWAP CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, zswap_max); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, zswap_writeback);#endif CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, private_id_objcg); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, private_id); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, kmemcg_id); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly, oom_group);
CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_read_mostly, memcg); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_read_mostly, lruvec_stats_percpu); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_read_mostly, lruvec_stats); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_read_mostly, shrinker_info); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_read_mostly, objcg);
CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_lruvec, lruvec);
CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_write_hot, lru_zone_size); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_write_hot, iter);#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_write_hot, slab_reclaimable); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_write_hot, slab_unreclaimable);#endif
CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_cold, orig_objcg); CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node, memcg_pn_cold, objcg_list);}
int __init mem_cgroup_init(void){ unsigned int memcg_size; int cpu;
/* * Currently s32 type (can refer to struct batched_lruvec_stat) is * used for per-memcg-per-cpu caching of per-node statistics. In order * to work fine, we should make sure that the overfill threshold can't * exceed S32_MAX / PAGE_SIZE. */ BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
memcg_struct_check();
cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, memcg_hotplug_cpu_dead);
memcg_wq = alloc_workqueue("memcg", WQ_PERCPU, 0); WARN_ON(!memcg_wq);
for_each_possible_cpu(cpu) { INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, drain_local_memcg_stock); INIT_WORK(&per_cpu_ptr(&obj_stock, cpu)->work, drain_local_obj_stock); }
memcg_size = struct_size_t(struct mem_cgroup, nodeinfo, nr_node_ids); memcg_cachep = kmem_cache_create("mem_cgroup", memcg_size, 0, SLAB_PANIC | SLAB_HWCACHE_ALIGN, NULL);
memcg_pn_cachep = KMEM_CACHE(mem_cgroup_per_node, SLAB_PANIC | SLAB_HWCACHE_ALIGN);
shrinker_unit_cache_init();
return 0;}
#ifdef CONFIG_SWAP/** * __mem_cgroup_try_charge_swap - try charging swap space for a folio * @folio: folio being added to swap * * Try to charge @folio's memcg for the swap space at folio->swap. * * Returns 0 on success, -ENOMEM on failure. */int __mem_cgroup_try_charge_swap(struct folio *folio){ unsigned int nr_pages = folio_nr_pages(folio); struct swap_cluster_info *ci; struct page_counter *counter; struct mem_cgroup *memcg; struct obj_cgroup *objcg; unsigned short private_id;
if (do_memsw_account()) return 0;
objcg = folio_objcg(folio); VM_WARN_ON_ONCE_FOLIO(!objcg, folio); if (!objcg) return 0;
scoped_guard(rcu) { memcg = obj_cgroup_memcg(objcg); if (!folio_test_swapcache(folio)) { memcg_memory_event(memcg, MEMCG_SWAP_FAIL); return 0; }
private_id = mem_cgroup_private_id_get(memcg, nr_pages);
if (!mem_cgroup_private_id_is_root(private_id) && !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { memcg_memory_event(memcg, MEMCG_SWAP_MAX); memcg_memory_event(memcg, MEMCG_SWAP_FAIL); mem_cgroup_private_id_put(private_id, nr_pages); return -ENOMEM; } mod_memcg_state(memcg, MEMCG_SWAP, nr_pages); }
ci = swap_cluster_get_and_lock(folio); __swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_pages, private_id); swap_cluster_unlock(ci);
return 0;}
/** * __mem_cgroup_uncharge_swap - uncharge swap space * @id: cgroup id to uncharge * @nr_pages: the amount of swap space to uncharge */void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages){ struct obj_cgroup *objcg; struct mem_cgroup *memcg;
rcu_read_lock(); objcg = obj_cgroup_from_private_id(id); memcg = obj_cgroup_memcg(objcg); if (memcg) { if (!mem_cgroup_private_id_is_root(id)) { if (do_memsw_account()) page_counter_uncharge(&memcg->memsw, nr_pages); else page_counter_uncharge(&memcg->swap, nr_pages); } mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); __mem_cgroup_private_id_put(objcg, id, nr_pages); } rcu_read_unlock();}
long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg){ long nr_swap_pages = get_nr_swap_pages();
if (!mem_cgroup_disabled() && !do_memsw_account()) nr_swap_pages = min(nr_swap_pages, page_counter_margin(&memcg->swap));
return nr_swap_pages;}
/** * mem_cgroup_get_folio_swap_margin - get a folio's memcg swap margin * @folio: folio whose memcg margin is queried * * Return: Remaining chargeable pages in the folio's memcg hierarchy. */long mem_cgroup_get_folio_swap_margin(const struct folio *folio){ struct mem_cgroup *memcg; long margin;
if (mem_cgroup_disabled() || do_memsw_account() || !folio_memcg_charged(folio)) return PAGE_COUNTER_MAX;
rcu_read_lock(); memcg = folio_memcg(folio); margin = page_counter_margin(&memcg->swap); rcu_read_unlock();
return margin;}
bool mem_cgroup_swap_full(const struct folio *folio){ struct mem_cgroup *memcg; bool ret = false;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (vm_swap_full()) return true; if (do_memsw_account() || !folio_memcg_charged(folio)) return ret;
rcu_read_lock(); memcg = folio_memcg(folio); for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) { unsigned long usage = page_counter_read(&memcg->swap);
if (usage * 2 >= READ_ONCE(memcg->swap.high) || usage * 2 >= READ_ONCE(memcg->swap.max)) { ret = true; break; } } rcu_read_unlock();
return ret;}
static int __init setup_swap_account(char *s){ bool res;
if (!kstrtobool(s, &res) && !res) pr_warn_once("The swapaccount=0 commandline option is deprecated " "in favor of configuring swap control via cgroupfs. " "Please report your usecase to linux-mm@kvack.org if you " "depend on this functionality.\n"); return 1;}__setup("swapaccount=", setup_swap_account);
static u64 swap_current_read(struct cgroup_subsys_state *css, struct cftype *cft){ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;}
static int swap_peak_show(struct seq_file *sf, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
guard(spinlock)(&memcg->peaks_lock); return peak_show(sf, v, &memcg->swap);}
static ssize_t swap_peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
return peak_write(of, buf, nbytes, off, &memcg->swap, &memcg->swap_peaks);}
static int swap_high_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->swap.high));}
static ssize_t swap_high_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned long high; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &high); if (err) return err;
page_counter_set_high(&memcg->swap, high);
return nbytes;}
static int swap_max_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->swap.max));}
static ssize_t swap_max_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned long max; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &max); if (err) return err;
xchg(&memcg->swap.max, max);
return nbytes;}
static int swap_events_show(struct seq_file *m, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
seq_printf(m, "high %lu\n", atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH])); seq_printf(m, "max %lu\n", atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX])); seq_printf(m, "fail %lu\n", atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
return 0;}
static struct cftype swap_files[] = { { .name = "swap.current", .flags = CFTYPE_NOT_ON_ROOT, .read_u64 = swap_current_read, }, { .name = "swap.high", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = swap_high_show, .write = swap_high_write, }, { .name = "swap.max", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = swap_max_show, .write = swap_max_write, }, { .name = "swap.peak", .flags = CFTYPE_NOT_ON_ROOT, .open = peak_open, .release = peak_release, .seq_show = swap_peak_show, .write = swap_peak_write, }, { .name = "swap.events", .flags = CFTYPE_NOT_ON_ROOT, .file_offset = offsetof(struct mem_cgroup, swap_events_file), .seq_show = swap_events_show, }, { } /* terminate */};
#ifdef CONFIG_ZSWAP/** * obj_cgroup_may_zswap - check if this cgroup can zswap * @objcg: the object cgroup * * Check if the hierarchical zswap limit has been reached. * * This doesn't check for specific headroom, and it is not atomic * either. But with zswap, the size of the allocation is only known * once compression has occurred, and this optimistic pre-check avoids * spending cycles on compression when there is already no room left * or zswap is disabled altogether somewhere in the hierarchy. */bool obj_cgroup_may_zswap(struct obj_cgroup *objcg){ struct mem_cgroup *memcg, *original_memcg; bool ret = true;
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return true;
original_memcg = get_mem_cgroup_from_objcg(objcg); for (memcg = original_memcg; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) { unsigned long max = READ_ONCE(memcg->zswap_max); unsigned long pages;
if (max == PAGE_COUNTER_MAX) continue; if (max == 0) { ret = false; break; }
/* Force flush to get accurate stats for charging */ __mem_cgroup_flush_stats(memcg, true); pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE; if (pages < max) continue; ret = false; break; } mem_cgroup_put(original_memcg); return ret;}
/** * obj_cgroup_charge_zswap - charge compression backend memory * @objcg: the object cgroup * @size: size of compressed object * * This forces the charge after obj_cgroup_may_zswap() allowed * compression and storage in zswap for this cgroup to go ahead. */void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size){ struct mem_cgroup *memcg;
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return;
if (obj_cgroup_is_root(objcg)) return;
VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC));
/* PF_MEMALLOC context, charging must succeed */ if (obj_cgroup_charge(objcg, GFP_KERNEL, size)) VM_WARN_ON_ONCE(1);
rcu_read_lock(); memcg = obj_cgroup_memcg(objcg); mod_memcg_state(memcg, MEMCG_ZSWAP_B, size); mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1); if (size == PAGE_SIZE) mod_memcg_state(memcg, MEMCG_ZSWAP_INCOMP, 1); rcu_read_unlock();}
/** * obj_cgroup_uncharge_zswap - uncharge compression backend memory * @objcg: the object cgroup * @size: size of compressed object * * Uncharges zswap memory on page in. */void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size){ struct mem_cgroup *memcg;
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return;
if (obj_cgroup_is_root(objcg)) return;
obj_cgroup_uncharge(objcg, size);
rcu_read_lock(); memcg = obj_cgroup_memcg(objcg); mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size); mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1); if (size == PAGE_SIZE) mod_memcg_state(memcg, MEMCG_ZSWAP_INCOMP, -1); rcu_read_unlock();}
bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg){ /* if zswap is disabled, do not block pages going to the swapping device */ if (!zswap_is_enabled()) return true;
for (; memcg; memcg = parent_mem_cgroup(memcg)) if (!READ_ONCE(memcg->zswap_writeback)) return false;
return true;}
static u64 zswap_current_read(struct cgroup_subsys_state *css, struct cftype *cft){ struct mem_cgroup *memcg = mem_cgroup_from_css(css);
mem_cgroup_flush_stats(memcg); return memcg_page_state(memcg, MEMCG_ZSWAP_B);}
static int zswap_max_show(struct seq_file *m, void *v){ return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->zswap_max));}
static ssize_t zswap_max_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); unsigned long max; int err;
buf = strstrip(buf); err = page_counter_memparse(buf, "max", &max); if (err) return err;
xchg(&memcg->zswap_max, max);
return nbytes;}
static int zswap_writeback_show(struct seq_file *m, void *v){ struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
seq_printf(m, "%d\n", READ_ONCE(memcg->zswap_writeback)); return 0;}
static ssize_t zswap_writeback_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off){ struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); int zswap_writeback; ssize_t parse_ret = kstrtoint(strstrip(buf), 0, &zswap_writeback);
if (parse_ret) return parse_ret;
if (zswap_writeback != 0 && zswap_writeback != 1) return -EINVAL;
WRITE_ONCE(memcg->zswap_writeback, zswap_writeback); return nbytes;}
static struct cftype zswap_files[] = { { .name = "zswap.current", .flags = CFTYPE_NOT_ON_ROOT, .read_u64 = zswap_current_read, }, { .name = "zswap.max", .flags = CFTYPE_NOT_ON_ROOT, .seq_show = zswap_max_show, .write = zswap_max_write, }, { .name = "zswap.writeback", .seq_show = zswap_writeback_show, .write = zswap_writeback_write, }, { } /* terminate */};#endif /* CONFIG_ZSWAP */
static int __init mem_cgroup_swap_init(void){ if (mem_cgroup_disabled()) return 0;
WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));#ifdef CONFIG_MEMCG_V1 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));#endif#ifdef CONFIG_ZSWAP WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, zswap_files));#endif return 0;}subsys_initcall(mem_cgroup_swap_init);
#endif /* CONFIG_SWAP */
void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask){ nodemask_t allowed;
if (!memcg) return;
/* * Since this interface is intended for use by migration paths, and * reclaim and migration are subject to race conditions such as changes * in effective_mems and hot-unpluging of nodes, inaccurate allowed * mask is acceptable. */ cpuset_nodes_allowed(memcg->css.cgroup, &allowed); nodes_and(*mask, *mask, allowed);}
void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg){ if (mem_cgroup_disabled() || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) return;
if (!memcg) memcg = root_mem_cgroup;
pr_warn("Memory cgroup min protection %lukB -- low protection %lukB", K(atomic_long_read(&memcg->memory.children_min_usage)), K(atomic_long_read(&memcg->memory.children_low_usage)));}