Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834/* SPDX-License-Identifier: GPL-2.0 *//* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo <tj@kernel.org> * Copyright (c) 2022 David Vernet <dvernet@meta.com> */#include <linux/bitmap.h>#include <linux/btf_ids.h>#include <linux/rhashtable.h>#include <linux/sched/clock.h>#include <linux/sched/isolation.h>#include <linux/suspend.h>#include <linux/sysrq.h>
#include "../pelt.h"#include "internal.h"#include "cid.h"#include "arena.h"#include "idle.h"
static DEFINE_RAW_SPINLOCK(scx_sched_lock);
/* * NOTE: sched_ext is in the process of growing multiple scheduler support and * scx_root usage is in a transitional state. Naked dereferences are safe if the * caller is one of the tasks attached to SCX and explicit RCU dereference is * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but * are used as temporary markers to indicate that the dereferences need to be * updated to point to the associated scheduler instances rather than scx_root. */struct scx_sched __rcu *scx_root;
/* * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock. * Readers can hold either or rcu_read_lock(). */static LIST_HEAD(scx_sched_all);
#ifdef CONFIG_EXT_SUB_SCHEDstatic const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), .insecure_elasticity = true, /* inserted under scx_sched_lock */};
static struct rhashtable scx_sched_hash;#endif
/* see SCX_OPS_TID_TO_TASK */static const struct rhashtable_params scx_tid_hash_params = { .key_len = sizeof_field(struct sched_ext_entity, tid), .key_offset = offsetof(struct sched_ext_entity, tid), .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */};static struct rhashtable scx_tid_hash;
/* * During exit, a task may schedule after losing its PIDs. When disabling the * BPF scheduler, we need to be able to iterate tasks in every state to * guarantee system safety. Maintain a dedicated task list which contains every * task between its fork and eventual free. */static DEFINE_RAW_SPINLOCK(scx_tasks_lock);static LIST_HEAD(scx_tasks);
/* ops enable/disable */static DEFINE_MUTEX(scx_enable_mutex);DEFINE_STATIC_KEY_FALSE(__scx_enabled);DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);static DEFINE_RAW_SPINLOCK(scx_bypass_lock);static bool scx_init_task_enabled;static bool scx_switching_all;DEFINE_STATIC_KEY_FALSE(__scx_switched_all);static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled);
/* * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler * and the tid->task table is live. Wraps the static key so callers don't * take the address, and hints "likely enabled" for the common case where * the feature is in use. */static inline bool scx_tid_to_task_enabled(void){ return static_branch_likely(&__scx_tid_to_task_enabled);}
static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);
#ifdef CONFIG_EXT_SUB_SCHED/* * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit * tasks for the sub-sched being enabled. Use a global variable instead of a * per-task field as all enables are serialized. */static struct scx_sched *scx_enabling_sub_sched;#else#define scx_enabling_sub_sched (struct scx_sched *)NULL#endif /* CONFIG_EXT_SUB_SCHED */
/* * A monotonically increasing sequence number that is incremented every time a * scheduler is enabled. This can be used to check if any custom sched_ext * scheduler has ever been used in the system. */static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0);
/* * Watchdog interval. All scx_sched's share a single watchdog timer and the * interval is half of the shortest sch->watchdog_timeout. */static unsigned long scx_watchdog_interval;
/* * The last time the delayed work was run. This delayed work relies on * ksoftirqd being able to run to service timer interrupts, so it's possible * that this work itself could get wedged. To account for this, we check that * it's not stalled in the timer tick, and trigger an error if it is. */static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES;
static struct delayed_work scx_watchdog_work;
/* * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated * lazily when enabling and freed when disabling to avoid waste when sched_ext * isn't active. */struct scx_kick_syncs { struct rcu_head rcu; unsigned long syncs[];};
static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs);
/* * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without * further synchronization. See scx_alloc_tid(). */struct scx_tid_alloc { u64 next; u64 end;};static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
/* * Direct dispatch marker. * * Non-NULL values are used for direct dispatch from enqueue path. A valid * pointer points to the task currently being enqueued. An ERR_PTR value is used * to indicate that direct dispatch has already happened. */static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node),};
static LLIST_HEAD(dsqs_to_free);
/* string formatting from BPF */struct scx_bstr_buf { u64 data[MAX_BPRINTF_VARARGS]; char line[SCX_EXIT_MSG_LEN];};
static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);static struct scx_bstr_buf scx_exit_bstr_buf;
/* ops debug dump */static DEFINE_RAW_SPINLOCK(scx_dump_lock);
struct scx_dump_data { s32 cpu; bool first; s32 cursor; struct seq_buf *s; const char *prefix; struct scx_bstr_buf buf;};
static struct scx_dump_data scx_dump_data = { .cpu = -1,};
/* /sys/kernel/sched_ext interface */static struct kset *scx_kset;
/* * Parameters that can be adjusted through /sys/module/sched_ext/parameters. * There usually is no reason to modify these as normal scheduler operation * shouldn't be affected by them. The knobs are primarily for debugging. */static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC;static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US;
static int set_slice_us(const char *val, const struct kernel_param *kp){ return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC);}
static const struct kernel_param_ops slice_us_param_ops = { .set = set_slice_us, .get = param_get_uint,};
static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp){ return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC);}
static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { .set = set_bypass_lb_intv_us, .get = param_get_uint,};
#undef MODULE_PARAM_PREFIX#define MODULE_PARAM_PREFIX "sched_ext."
module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600);MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)");module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600);MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)");
#undef MODULE_PARAM_PREFIX
#define CREATE_TRACE_POINTS#include <trace/events/sched_ext.h>
static void run_deferred(struct rq *rq);static bool task_dead_and_done(struct task_struct *p);static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, ...){ va_list args; bool ret;
va_start(args, fmt); ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args); va_end(args);
return ret;}
static long jiffies_delta_msecs(unsigned long at, unsigned long now){ if (time_after(at, now)) return jiffies_to_msecs(at - now); else return -(long)jiffies_to_msecs(now - at);}
static bool u32_before(u32 a, u32 b){ return (s32)(a - b) < 0;}
#ifdef CONFIG_EXT_SUB_SCHED/** * scx_next_descendant_pre - find the next descendant for pre-order walk * @pos: the current position (%NULL to initiate traversal) * @root: sched whose descendants to walk * * To be used by scx_for_each_descendant_pre(). Find the next descendant to * visit for pre-order traversal of @root's descendants. @root is included in * the iteration and the first node to be visited. */static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root){ struct scx_sched *next;
lockdep_assert(lockdep_is_held(&scx_enable_mutex) || lockdep_is_held(&scx_sched_lock));
/* if first iteration, visit @root */ if (!pos) return root;
/* visit the first child if exists */ next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); if (next) return next;
/* no child, visit my or the closest ancestor's next sibling */ while (pos != root) { if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) return list_next_entry(pos, sibling); pos = scx_parent(pos); }
return NULL;}
static struct scx_sched *scx_find_sub_sched(u64 cgroup_id){ return rhashtable_lookup(&scx_sched_hash, &cgroup_id, scx_sched_hash_params);}
static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch){ rcu_assign_pointer(p->scx.sched, sch);}#else /* CONFIG_EXT_SUB_SCHED */static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}#endif /* CONFIG_EXT_SUB_SCHED */
/** * scx_is_descendant - Test whether sched is a descendant * @sch: sched to test * @ancestor: ancestor sched to test against * * Test whether @sch is a descendant of @ancestor. */static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor){ if (sch->level < ancestor->level) return false; return sch->ancestors[ancestor->level] == ancestor;}
/** * scx_for_each_descendant_pre - pre-order walk of a sched's descendants * @pos: iteration cursor * @root: sched to walk the descendants of * * Walk @root's descendants. @root is included in the iteration and the first * node to be visited. Must be called with either scx_enable_mutex or * scx_sched_lock held. */#define scx_for_each_descendant_pre(pos, root) \ for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ (pos) = scx_next_descendant_pre((pos), (root)))
static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu){ return &sch->pnode[cpu_to_node(cpu)]->global_dsq;}
static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id){ return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params);}
static const struct sched_class *scx_setscheduler_class(struct task_struct *p){ if (p->sched_class == &stop_sched_class) return &stop_sched_class;
return __setscheduler_class(p->policy, p->prio);}
static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu){ return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;}
static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu){#ifdef CONFIG_EXT_SUB_SCHED /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The * not-bypassing ancestor is responsible for scheduling all tasks from * bypassing sub-trees. If all ancestors including root are bypassing, * all tasks should go to the root's bypass DSQs. * * Whenever a sched starts bypassing, all runnable tasks in its subtree * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. */ while (scx_parent(sch) && scx_bypassing(sch, cpu)) sch = scx_parent(sch);#endif /* CONFIG_EXT_SUB_SCHED */
return bypass_dsq(sch, cpu);}
/** * bypass_dsp_enabled - Check if bypass dispatch path is enabled * @sch: scheduler to check * * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors * are bypassing. In the former case, the ancestor is not itself bypassing but * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, * the ancestor's bypass dispatch path must be active even though its own * bypass_depth remains zero. * * This function checks bypass_dsp_enable_depth which is managed separately from * bypass_depth to enable this decoupling. See enable_bypass_dsp() and * disable_bypass_dsp(). */static bool bypass_dsp_enabled(struct scx_sched *sch){ return unlikely(atomic_read(&sch->bypass_dsp_enable_depth));}
/** * rq_is_open - Is the rq available for immediate execution of an SCX task? * @rq: rq to test * @enq_flags: optional %SCX_ENQ_* of the task being enqueued * * Returns %true if @rq is currently open for executing an SCX task. After a * %false return, @rq is guaranteed to invoke SCX dispatch path at least once * before going to idle and not inserting a task into @rq's local DSQ after a * %false return doesn't cause @rq to stall. */static bool rq_is_open(struct rq *rq, u64 enq_flags){ lockdep_assert_rq_held(rq);
/* * A higher-priority class task is either running or in the process of * waking up on @rq. */ if (sched_class_above(rq->next_class, &ext_sched_class)) return false;
/* * @rq is either in transition to or in idle and there is no * higher-priority class task waking up on it. */ if (sched_class_above(&ext_sched_class, rq->next_class)) return true;
/* * @rq is either picking, in transition to, or running an SCX task. */
/* * If we're in the dispatch path holding rq lock, $curr may or may not * be ready depending on whether the on-going dispatch decides to extend * $curr's slice. We say yes here and resolve it at the end of dispatch. * See balance_one(). */ if (rq->scx.flags & SCX_RQ_IN_BALANCE) return true;
/* * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ if (enq_flags & SCX_ENQ_PREEMPT) return true;
/* * @rq is either in transition to or running an SCX task and can't go * idle without another SCX dispatch cycle. */ return false;}
/* * Track the rq currently locked. * * This allows kfuncs to safely operate on rq from any scx ops callback, * knowing which rq is already locked. */DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
static void switch_rq_lock(struct rq *from, struct rq *to){ bool tracked = scx_locked_rq() == from;
if (tracked) update_locked_rq(NULL); raw_spin_rq_unlock(from); raw_spin_rq_lock(to); if (tracked) update_locked_rq(to);}
/* * Flipped on enable per sch->is_cid_type. Declared in internal.h so * subsystem inlines can read it. */DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
/** * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task * @sch: scx_sched being invoked * @rq: rq to update as the currently-locked rq, or NULL * @task: task whose affinity is changing * @cpumask: new cpumask * * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu * scratch in cid.c and dispatch through the ops_cid union view. Caller * must hold @rq's rq lock so this_cpu_ptr is stable across the call. */static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask){ WARN_ON_ONCE(current->scx.kf_tasks[0]); current->scx.kf_tasks[0] = task; if (rq) update_locked_rq(rq);
if (scx_is_cid_type()) { struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); /* * Build the per-CPU arena cmask and hand BPF its arena address. * Caller holds the rq lock with IRQs disabled, which makes us * the sole user of the scratch area. */ scx_cpumask_to_cmask(cpumask, kern_va); sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va)); } else { sch->ops.set_cpumask(task, cpumask); }
if (rq) update_locked_rq(NULL); current->scx.kf_tasks[0] = NULL;}
enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16,
__SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31,
__SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME,};
/** * nldsq_next_task - Iterate to the next task in a non-local DSQ * @dsq: non-local dsq being iterated * @cur: current position, %NULL to start iteration * @rev: walk backwards * * Returns %NULL when iteration is finished. */static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, struct task_struct *cur, bool rev){ struct list_head *list_node; struct scx_dsq_list_node *dsq_lnode;
lockdep_assert_held(&dsq->lock);
if (cur) list_node = &cur->scx.dsq_list.node; else list_node = &dsq->list;
/* find the next task, need to skip BPF iteration cursors */ do { if (rev) list_node = list_node->prev; else list_node = list_node->next;
if (list_node == &dsq->list) return NULL;
dsq_lnode = container_of(list_node, struct scx_dsq_list_node, node); } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR);
return container_of(dsq_lnode, struct task_struct, scx.dsq_list);}
#define nldsq_for_each_task(p, dsq) \ for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ (p) = nldsq_next_task((dsq), (p), false))
/** * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() * @dsq: non-local dsq being iterated * * Find the next task in a cursor based iteration. The caller must have * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock * between the iteration steps. * * Only tasks which were queued before @cursor was initialized are visible. This * bounds the iteration and guarantees that vtime never jumps in the other * direction while iterating. */static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor, struct scx_dispatch_q *dsq){ bool rev = cursor->flags & SCX_DSQ_ITER_REV; struct task_struct *p;
lockdep_assert_held(&dsq->lock); BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR));
if (list_empty(&cursor->node)) p = NULL; else p = container_of(cursor, struct task_struct, scx.dsq_list);
/* skip cursors and tasks that were queued after @cursor init */ do { p = nldsq_next_task(dsq, p, rev); } while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq)));
if (p) { if (rev) list_move_tail(&cursor->node, &p->scx.dsq_list.node); else list_move(&cursor->node, &p->scx.dsq_list.node); } else { list_del_init(&cursor->node); }
return p;}
/** * nldsq_cursor_lost_task - Test whether someone else took the task since iteration * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR() * @rq: rq @p was on * @dsq: dsq @p was on * @p: target task * * @p is a task returned by nldsq_cursor_next_task(). The locks may have been * dropped and re-acquired inbetween. Verify that no one else took or is in the * process of taking @p from @dsq. * * On %false return, the caller can assume full ownership of @p. */static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p){ lockdep_assert_rq_held(rq); lockdep_assert_held(&dsq->lock);
/* * @p could have already left $src_dsq, got re-enqueud, or be in the * process of being consumed by someone else. */ if (unlikely(p->scx.dsq != dsq || u32_before(cursor->priv, p->scx.dsq_seq) || p->scx.holding_cpu >= 0)) return true;
/* if @p has stayed on @dsq, its rq couldn't have changed */ if (WARN_ON_ONCE(rq != task_rq(p))) return true;
return false;}
/* * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] * dispatch order. BPF-visible iterator is opaque and larger to allow future * changes without breaking backward compatibility. Can be used with * bpf_for_each(). See bpf_iter_scx_dsq_*(). */struct bpf_iter_scx_dsq_kern { struct scx_dsq_list_node cursor; struct scx_dispatch_q *dsq; u64 slice; u64 vtime;} __attribute__((aligned(8)));
struct bpf_iter_scx_dsq { u64 __opaque[6];} __attribute__((aligned(8)));
static u32 scx_get_task_state(const struct task_struct *p){ return p->scx.flags & SCX_TASK_STATE_MASK;}
static void scx_set_task_state(struct task_struct *p, u32 state){ u32 prev_state = scx_get_task_state(p); bool warn = false;
switch (state) { case SCX_TASK_NONE: warn = prev_state == SCX_TASK_DEAD; break; case SCX_TASK_INIT_BEGIN: warn = prev_state != SCX_TASK_NONE; break; case SCX_TASK_INIT: warn = prev_state != SCX_TASK_INIT_BEGIN; p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; break; case SCX_TASK_READY: warn = !(prev_state == SCX_TASK_INIT || prev_state == SCX_TASK_ENABLED); break; case SCX_TASK_ENABLED: warn = prev_state != SCX_TASK_READY; break; case SCX_TASK_DEAD: warn = !(prev_state == SCX_TASK_NONE || prev_state == SCX_TASK_INIT_BEGIN); break; default: WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]", prev_state, state, p->comm, p->pid); return; }
WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]", prev_state, state, p->comm, p->pid);
p->scx.flags &= ~SCX_TASK_STATE_MASK; p->scx.flags |= state;}
/* * SCX task iterator. */struct scx_task_iter { struct sched_ext_entity cursor; struct task_struct *locked_task; struct rq *rq; struct rq_flags rf; u32 cnt; bool list_locked;#ifdef CONFIG_EXT_SUB_SCHED struct cgroup *cgrp; struct cgroup_subsys_state *css_pos; struct css_task_iter css_iter;#endif};
/** * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration * @iter: iterator to init * @cgrp: Optional root of cgroup subhierarchy to iterate * * Initialize @iter. Once initialized, @iter must eventually be stopped with * scx_task_iter_stop(). * * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks. * * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup * task migrations. * * The two modes of iterations are largely independent and it's likely that * scx_tasks can be removed in favor of always using cgroup iteration if * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS. * * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() * between this and the first next() call or between any two next() calls. If * the locks are released between two next() calls, the caller is responsible * for ensuring that the task being iterated remains accessible either through * RCU read lock or obtaining a reference count. * * All tasks which existed when the iteration started are guaranteed to be * visited as long as they are not dead. */static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp){ memset(iter, 0, sizeof(*iter));
#ifdef CONFIG_EXT_SUB_SCHED if (cgrp) { lockdep_assert_held(&cgroup_mutex); iter->cgrp = cgrp; iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self); css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, &iter->css_iter); return; }#endif raw_spin_lock_irq(&scx_tasks_lock);
iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; list_add(&iter->cursor.tasks_node, &scx_tasks); iter->list_locked = true;}
static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter){ if (iter->locked_task) { __balance_callbacks(iter->rq, &iter->rf); task_rq_unlock(iter->rq, iter->locked_task, &iter->rf); iter->locked_task = NULL; }}
/** * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator * @iter: iterator to unlock * * If @iter is in the middle of a locked iteration, it may be locking the rq of * the task currently being visited in addition to scx_tasks_lock. Unlock both. * This function can be safely called anytime during an iteration. The next * iterator operation will automatically restore the necessary locking. */static void scx_task_iter_unlock(struct scx_task_iter *iter){ __scx_task_iter_rq_unlock(iter); if (iter->list_locked) { iter->list_locked = false; raw_spin_unlock_irq(&scx_tasks_lock); }}
static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter){ if (!iter->list_locked) { raw_spin_lock_irq(&scx_tasks_lock); iter->list_locked = true; }}
/** * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq * @iter: iterator to relock * @p: task whose rq to lock, or %NULL for scx_tasks_lock only * * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once * re-acquired, both locks are managed by the iterator from here on. */static void scx_task_iter_relock(struct scx_task_iter *iter, struct task_struct *p){ __scx_task_iter_maybe_relock(iter); if (p) { iter->rq = task_rq_lock(p, &iter->rf); iter->locked_task = p; }}
/** * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock * @iter: iterator to exit * * Exit a previously initialized @iter. Must be called with scx_tasks_lock held * which is released on return. If the iterator holds a task's rq lock, that rq * lock is also released. See scx_task_iter_start() for details. */static void scx_task_iter_stop(struct scx_task_iter *iter){#ifdef CONFIG_EXT_SUB_SCHED if (iter->cgrp) { if (iter->css_pos) css_task_iter_end(&iter->css_iter); __scx_task_iter_rq_unlock(iter); return; }#endif __scx_task_iter_maybe_relock(iter); list_del_init(&iter->cursor.tasks_node); scx_task_iter_unlock(iter);}
/** * scx_task_iter_next - Next task * @iter: iterator to walk * * Visit the next task. See scx_task_iter_start() for details. Locks are dropped * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls * by holding scx_tasks_lock for too long. */static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter){ struct list_head *cursor = &iter->cursor.tasks_node; struct sched_ext_entity *pos;
if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) { scx_task_iter_unlock(iter); cond_resched(); }
#ifdef CONFIG_EXT_SUB_SCHED if (iter->cgrp) { while (iter->css_pos) { struct task_struct *p;
p = css_task_iter_next(&iter->css_iter); if (p) return p;
css_task_iter_end(&iter->css_iter); iter->css_pos = css_next_descendant_pre(iter->css_pos, &iter->cgrp->self); if (iter->css_pos) css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD, &iter->css_iter); } return NULL; }#endif __scx_task_iter_maybe_relock(iter);
list_for_each_entry(pos, cursor, tasks_node) { if (&pos->tasks_node == &scx_tasks) return NULL; if (!(pos->flags & SCX_TASK_CURSOR)) { list_move(cursor, &pos->tasks_node); return container_of(pos, struct task_struct, scx); } }
/* can't happen, should always terminate at scx_tasks above */ BUG();}
/** * scx_task_iter_next_locked - Next non-idle task with its rq locked * @iter: iterator to walk * * Visit the non-idle task with its rq lock held. Allows callers to specify * whether they would like to filter out dead tasks. See scx_task_iter_start() * for details. */static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter){ struct task_struct *p;
__scx_task_iter_rq_unlock(iter);
while ((p = scx_task_iter_next(iter))) { /* * scx_task_iter is used to prepare and move tasks into SCX * while loading the BPF scheduler and vice-versa while * unloading. The init_tasks ("swappers") should be excluded * from the iteration because: * * - It's unsafe to use __setschduler_prio() on an init_task to * determine the sched_class to use as it won't preserve its * idle_sched_class. * * - ops.init/exit_task() can easily be confused if called with * init_tasks as they, e.g., share PID 0. * * As init_tasks are never scheduled through SCX, they can be * skipped safely. Note that is_idle_task() which tests %PF_IDLE * doesn't work here: * * - %PF_IDLE may not be set for an init_task whose CPU hasn't * yet been onlined. * * - %PF_IDLE can be set on tasks that are not init_tasks. See * play_idle_precise() used by CONFIG_IDLE_INJECT. * * Test for idle_sched_class as only init_tasks are on it. */ if (p->sched_class == &idle_sched_class) continue;
iter->rq = task_rq_lock(p, &iter->rf); iter->locked_task = p;
/* * cgroup_task_dead() removes the dead tasks from cset->tasks * after sched_ext_dead() and cgroup iteration may see tasks * which already finished sched_ext_dead(). %SCX_TASK_DEAD is * set by sched_ext_dead() under @p's rq lock. Test it to * avoid visiting tasks which are already dead from SCX POV. */ if (scx_get_task_state(p) == SCX_TASK_DEAD) { __scx_task_iter_rq_unlock(iter); continue; }
return p; } return NULL;}
/** * scx_add_event - Increase an event counter for 'name' by 'cnt' * @sch: scx_sched to account events for * @name: an event name defined in struct scx_event_stats * @cnt: the number of the event occurred * * This can be used when preemption is not disabled. */#define scx_add_event(sch, name, cnt) do { \ this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ trace_sched_ext_event(#name, (cnt)); \} while(0)
/** * __scx_add_event - Increase an event counter for 'name' by 'cnt' * @sch: scx_sched to account events for * @name: an event name defined in struct scx_event_stats * @cnt: the number of the event occurred * * This should be used only when preemption is disabled. */#define __scx_add_event(sch, name, cnt) do { \ __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ trace_sched_ext_event(#name, cnt); \} while(0)
/** * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' * @dst_e: destination event stats * @src_e: source event stats * @kind: a kind of event to be aggregated */#define scx_agg_event(dst_e, src_e, kind) do { \ (dst_e)->kind += READ_ONCE((src_e)->kind); \} while(0)
/** * scx_dump_event - Dump an event 'kind' in 'events' to 's' * @s: output seq_buf * @events: event stats * @kind: a kind of event to dump */#define scx_dump_event(s, events, kind) do { \ dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \} while (0)
static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events);
static enum scx_enable_state scx_enable_state(void){ return atomic_read(&scx_enable_state_var);}
static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to){ return atomic_xchg(&scx_enable_state_var, to);}
static bool scx_tryset_enable_state(enum scx_enable_state to, enum scx_enable_state from){ int from_v = from;
return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to);}
/** * wait_ops_state - Busy-wait the specified ops state to end * @p: target task * @opss: state to wait the end of * * Busy-wait for @p to transition out of @opss. This can only be used when the * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also * has load_acquire semantics to ensure that the caller can see the updates made * in the enqueueing and dispatching paths. */static void wait_ops_state(struct task_struct *p, unsigned long opss){ do { cpu_relax(); } while (atomic_long_read_acquire(&p->scx.ops_state) == opss);}
static inline bool __cpu_valid(s32 cpu){ return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu));}
/** * scx_cpu_valid - Verify a cpu number, to be used on ops input args * @sch: scx_sched to abort on error * @cpu: cpu number which came from a BPF ops * @where: extra information reported on error * * @cpu is a cpu number which came from the BPF scheduler and can be any value. * Verify that it is in range and one of the possible cpus. If invalid, trigger * an ops error. */bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where){ if (__cpu_valid(cpu)) { return true; } else { scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: ""); return false; }}
/** * ops_sanitize_err - Sanitize a -errno value * @sch: scx_sched to error out on error * @ops_name: operation to blame on failure * @err: -errno value to sanitize * * Verify @err is a valid -errno. If not, trigger scx_error() and return * -%EPROTO. This is necessary because returning a rogue -errno up the chain can * cause misbehaviors. For an example, a large negative return from * ops.init_task() triggers an oops when passed up the call chain because the * value fails IS_ERR() test after being encoded with ERR_PTR() and then is * handled as a pointer. */static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err){ if (err < 0 && err >= -MAX_ERRNO) return err;
scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); return -EPROTO;}
static void deferred_bal_cb_workfn(struct rq *rq){ run_deferred(rq);}
static void deferred_irq_workfn(struct irq_work *irq_work){ struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work);
raw_spin_rq_lock(rq); run_deferred(rq); raw_spin_rq_unlock(rq);}
/** * schedule_deferred - Schedule execution of deferred actions on an rq * @rq: target rq * * Schedule execution of deferred actions on @rq. Deferred actions are executed * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks * to other rqs. */static void schedule_deferred(struct rq *rq){ /* * This is the fallback when schedule_deferred_locked() can't use * the cheaper balance callback or wakeup hook paths (the target * CPU is not in balance or wakeup). Currently, this is primarily * hit by reenqueue operations targeting a remote CPU. * * Queue on the target CPU. The deferred work can run from any CPU * correctly - the _locked() path already processes remote rqs from * the calling CPU - but targeting the owning CPU allows IPI delivery * without waiting for the calling CPU to re-enable IRQs and is * cheaper as the reenqueue runs locally. */ irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq));}
/** * schedule_deferred_locked - Schedule execution of deferred actions on an rq * @rq: target rq * * Schedule execution of deferred actions on @rq. Equivalent to * schedule_deferred() but requires @rq to be locked and can be more efficient. */static void schedule_deferred_locked(struct rq *rq){ lockdep_assert_rq_held(rq);
/* * If in the middle of waking up a task, task_woken_scx() will be called * afterwards which will then run the deferred actions, no need to * schedule anything. */ if (rq->scx.flags & SCX_RQ_IN_WAKEUP) return;
/* Don't do anything if there already is a deferred operation. */ if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) return;
/* * If in balance, the balance callbacks will be called before rq lock is * released. Schedule one. * * * We can't directly insert the callback into the * rq's list: The call can drop its lock and make the pending balance * callback visible to unrelated code paths that call rq_pin_lock(). * * Just let balance_one() know that it must do it itself. */ if (rq->scx.flags & SCX_RQ_IN_BALANCE) { rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; return; }
/* * No scheduler hooks available. Use the generic irq_work path. The * above WAKEUP and BALANCE paths should cover most of the cases and the * time to IRQ re-enable shouldn't be long. */ schedule_deferred(rq);}
static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq){ struct rq *rq;
/* * Allowing reenqueues doesn't make sense while bypassing. This also * blocks from new reenqueues to be scheduled on dead scheds. */ if (unlikely(READ_ONCE(sch->bypass_depth))) return;
if (dsq->id == SCX_DSQ_LOCAL) { rq = container_of(dsq, struct rq, scx.local_dsq);
struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq)); struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local;
/* * Pairs with smp_mb() in process_deferred_reenq_locals() and * guarantees that there is a reenq_local() afterwards. */ smp_mb();
if (list_empty(&drl->node) || (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) {
guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
if (list_empty(&drl->node)) list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals); WRITE_ONCE(drl->flags, drl->flags | reenq_flags); } } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { rq = this_rq();
struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user;
/* * Pairs with smp_mb() in process_deferred_reenq_users() and * guarantees that there is a reenq_user() afterwards. */ smp_mb();
if (list_empty(&dru->node) || (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) {
guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
if (list_empty(&dru->node)) list_move_tail(&dru->node, &rq->scx.deferred_reenq_users); WRITE_ONCE(dru->flags, dru->flags | reenq_flags); } } else { scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); return; }
if (rq == locked_rq) schedule_deferred_locked(rq); else schedule_deferred(rq);}
static void schedule_reenq_local(struct rq *rq, u64 reenq_flags){ struct scx_sched *root = rcu_dereference_sched(scx_root);
if (WARN_ON_ONCE(!root)) return;
schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq);}
/** * touch_core_sched - Update timestamp used for core-sched task ordering * @rq: rq to read clock from, must be locked * @p: task to update the timestamp for * * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to * implement global or local-DSQ FIFO ordering for core-sched. Should be called * when a task becomes runnable and its turn on the CPU ends (e.g. slice * exhaustion). */static void touch_core_sched(struct rq *rq, struct task_struct *p){ lockdep_assert_rq_held(rq);
#ifdef CONFIG_SCHED_CORE /* * It's okay to update the timestamp spuriously. Use * sched_core_disabled() which is cheaper than enabled(). * * As this is used to determine ordering between tasks of sibling CPUs, * it may be better to use per-core dispatch sequence instead. */ if (!sched_core_disabled()) p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq));#endif}
/** * touch_core_sched_dispatch - Update core-sched timestamp on dispatch * @rq: rq to read clock from, must be locked * @p: task being dispatched * * If the BPF scheduler implements custom core-sched ordering via * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO * ordering within each local DSQ. This function is called from dispatch paths * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. */static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p){ lockdep_assert_rq_held(rq);
#ifdef CONFIG_SCHED_CORE if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) touch_core_sched(rq, p);#endif}
static void update_curr_scx(struct rq *rq){ struct task_struct *curr = rq->curr; s64 delta_exec;
delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return;
if (curr->scx.slice != SCX_SLICE_INF) { curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); if (!curr->scx.slice) touch_core_sched(rq, curr); }
dl_server_update(&rq->ext_server, delta_exec);}
static bool scx_dsq_priq_less(struct rb_node *node_a, const struct rb_node *node_b){ const struct task_struct *a = container_of(node_a, struct task_struct, scx.dsq_priq); const struct task_struct *b = container_of(node_b, struct task_struct, scx.dsq_priq);
return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime);}
static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags){ /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ WRITE_ONCE(dsq->nr, dsq->nr + 1);
/* * Once @p reaches a local DSQ, it can only leave it by being dispatched * to the CPU or dequeued. In both cases, the only way @p can go back to * the BPF sched is through enqueueing. If being inserted into a local * DSQ with IMMED, persist the state until the next enqueueing event in * do_enqueue_task() so that we can maintain IMMED protection through * e.g. SAVE/RESTORE cycles and slice extensions. */ if (enq_flags & SCX_ENQ_IMMED) { if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK)); return; } p->scx.flags |= SCX_TASK_IMMED; }
if (p->scx.flags & SCX_TASK_IMMED) { struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) return;
rq->scx.nr_immed++;
/* * If @rq already had other tasks or the current task is not * done yet, @p can't go on the CPU immediately. Re-enqueue. */ if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) schedule_reenq_local(rq, 0); }}
static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p){ /* see dsq_inc_nr() */ WRITE_ONCE(dsq->nr, dsq->nr - 1);
if (p->scx.flags & SCX_TASK_IMMED) { struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) || WARN_ON_ONCE(rq->scx.nr_immed <= 0)) return;
rq->scx.nr_immed--; }}
static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p){ p->scx.slice = READ_ONCE(sch->slice_dfl); __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);}
/* * Return true if @p is moving due to an internal SCX migration, false * otherwise. */static inline bool task_scx_migrating(struct task_struct *p){ /* * We only need to check sticky_cpu: it is set to the destination * CPU in move_remote_task_to_local_dsq() before deactivate_task() * and cleared when the task is enqueued on the destination, so it * is only non-negative during an internal SCX migration. */ return p->scx.sticky_cpu >= 0;}
/* * Call ops.dequeue() if the task is in BPF custody and not migrating. * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. */static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 deq_flags){ if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) return;
if (SCX_HAS_OP(sch, dequeue)) SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
p->scx.flags &= ~SCX_TASK_IN_CUSTODY;}
static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags){ struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
call_task_dequeue(sch, rq, p, 0);
/* * Note that @rq's lock may be dropped between this enqueue and @p * actually getting on CPU. This gives higher-class tasks (e.g. RT) * an opportunity to wake up on @rq and prevent @p from running. * Here are some concrete examples: * * Example 1: * * We dispatch two tasks from a single ops.dispatch(): * - First, a local task to this CPU's local DSQ; * - Second, a local/remote task to a remote CPU's local DSQ. * We must drop the local rq lock in order to finish the second * dispatch. In that time, an RT task can wake up on the local rq. * * Example 2: * * We dispatch a local/remote task to a remote CPU's local DSQ. * We must drop the remote rq lock before the dispatched task can run, * which gives an RT task an opportunity to wake up on the remote rq. * * Both examples work the same if we replace dispatching with moving * the tasks from a user-created DSQ. * * We must detect these wakeups so that we can re-enqueue IMMED tasks * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this * purpose, but for it to be invoked, we must ensure that we bump * @rq->next_class to &ext_sched_class if it's currently idle. * * wakeup_preempt() does the bumping, and since we only invoke it if * @rq->next_class is below &ext_sched_class, it will also * resched_curr(rq). */ if (sched_class_above(p->sched_class, rq->next_class)) wakeup_preempt(rq, p, 0);
/* * If @rq is in balance, the CPU is already vacant and looking for the * next task to run. No need to preempt or trigger resched after moving * @p into its local DSQ. * Note that the wakeup_preempt() above may have already triggered * a resched if @rq->next_class was idle. It's harmless, since * need_resched is cleared immediately after task pick. */ if (rq->scx.flags & SCX_RQ_IN_BALANCE) return;
if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { rq->curr->scx.slice = 0; resched_curr(rq); }}
static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags){ bool is_local = dsq->id == SCX_DSQ_LOCAL;
WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || !RB_EMPTY_NODE(&p->scx.dsq_priq));
if (!is_local) { raw_spin_lock_nested(&dsq->lock, (enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0);
if (unlikely(dsq->id == SCX_DSQ_INVALID)) { scx_error(sch, "attempting to dispatch to a destroyed dsq"); /* fall back to the global dsq */ raw_spin_unlock(&dsq->lock); dsq = find_global_dsq(sch, task_cpu(p)); raw_spin_lock(&dsq->lock); } }
if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && (enq_flags & SCX_ENQ_DSQ_PRIQ))) { /* * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from * their FIFO queues. To avoid confusion and accidentally * starving vtime-dispatched tasks by FIFO-dispatched tasks, we * disallow any internal DSQ from doing vtime ordering of * tasks. */ scx_error(sch, "cannot use vtime ordering for built-in DSQs"); enq_flags &= ~SCX_ENQ_DSQ_PRIQ; }
if (enq_flags & SCX_ENQ_DSQ_PRIQ) { struct rb_node *rbp;
/* * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are * linked to both the rbtree and list on PRIQs, this can only be * tested easily when adding the first task. */ if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && nldsq_next_task(dsq, NULL, false))) scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks", dsq->id);
p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less);
/* * Find the previous task and insert after it on the list so * that @dsq->list is vtime ordered. */ rbp = rb_prev(&p->scx.dsq_priq); if (rbp) { struct task_struct *prev = container_of(rbp, struct task_struct, scx.dsq_priq); list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); /* first task unchanged - no update needed */ } else { list_add(&p->scx.dsq_list.node, &dsq->list); /* not builtin and new task is at head - use fastpath */ rcu_assign_pointer(dsq->first_task, p); } } else { /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", dsq->id);
if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { list_add(&p->scx.dsq_list.node, &dsq->list); /* new task inserted at head - use fastpath */ if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } else { /* * dsq->list can contain parked BPF iterator cursors, so * list_empty() here isn't a reliable proxy for "no real * task in the DSQ". Test dsq->first_task directly. */ list_add_tail(&p->scx.dsq_list.node, &dsq->list); if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } }
/* seq records the order tasks are queued, used by BPF DSQ iterator */ WRITE_ONCE(dsq->seq, dsq->seq + 1); p->scx.dsq_seq = dsq->seq;
dsq_inc_nr(dsq, p, enq_flags); p->scx.dsq = dsq;
/* * Update custody and call ops.dequeue() before clearing ops_state: * once ops_state is cleared, waiters in ops_dequeue() can proceed * and dequeue_task_scx() will RMW p->scx.flags. If we clear * ops_state first, both sides would modify p->scx.flags * concurrently in a non-atomic way. */ if (is_local) { local_dsq_post_enq(sch, dsq, p, enq_flags); } else { /* * Task on global/bypass DSQ: leave custody, task on * non-terminal DSQ: enter custody. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) call_task_dequeue(sch, rq, p, 0); else p->scx.flags |= SCX_TASK_IN_CUSTODY;
raw_spin_unlock(&dsq->lock); }
/* * We're transitioning out of QUEUEING or DISPATCHING. store_release to * match waiters' load_acquire. */ if (enq_flags & SCX_ENQ_CLEAR_OPSS) atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);}
static void task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq){ WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { rb_erase(&p->scx.dsq_priq, &dsq->priq); RB_CLEAR_NODE(&p->scx.dsq_priq); p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; }
list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p);
if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && dsq->first_task == p) { struct task_struct *first_task;
first_task = nldsq_next_task(dsq, NULL, false); rcu_assign_pointer(dsq->first_task, first_task); }}
static void dispatch_dequeue(struct rq *rq, struct task_struct *p){ struct scx_dispatch_q *dsq = p->scx.dsq; bool is_local = dsq == &rq->scx.local_dsq;
lockdep_assert_rq_held(rq);
if (!dsq) { /* * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. * Unlinking is all that's needed to cancel. */ if (unlikely(!list_empty(&p->scx.dsq_list.node))) list_del_init(&p->scx.dsq_list.node);
/* * When dispatching directly from the BPF scheduler to a local * DSQ, the task isn't associated with any DSQ but * @p->scx.holding_cpu may be set under the protection of * %SCX_OPSS_DISPATCHING. */ if (p->scx.holding_cpu >= 0) p->scx.holding_cpu = -1;
return; }
if (!is_local) raw_spin_lock(&dsq->lock);
/* * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't * change underneath us. */ if (p->scx.holding_cpu < 0) { /* @p must still be on @dsq, dequeue */ task_unlink_from_dsq(p, dsq); } else { /* * We're racing against dispatch_to_local_dsq() which already * removed @p from @dsq and set @p->scx.holding_cpu. Clear the * holding_cpu which tells dispatch_to_local_dsq() that it lost * the race. */ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); p->scx.holding_cpu = -1; } p->scx.dsq = NULL;
if (!is_local) raw_spin_unlock(&dsq->lock);}
/* * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq * and dsq are locked. */static void dispatch_dequeue_locked(struct task_struct *p, struct scx_dispatch_q *dsq){ lockdep_assert_rq_held(task_rq(p)); lockdep_assert_held(&dsq->lock);
task_unlink_from_dsq(p, dsq); p->scx.dsq = NULL;}
static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch, struct rq *rq, u64 dsq_id, s32 tcpu){ struct scx_dispatch_q *dsq;
if (dsq_id == SCX_DSQ_LOCAL) return &rq->scx.local_dsq;
if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) return find_global_dsq(sch, tcpu);
return &cpu_rq(cpu)->scx.local_dsq; }
if (dsq_id == SCX_DSQ_GLOBAL) dsq = find_global_dsq(sch, tcpu); else dsq = find_user_dsq(sch, dsq_id);
if (unlikely(!dsq)) { scx_error(sch, "non-existent DSQ 0x%llx", dsq_id); return find_global_dsq(sch, tcpu); }
return dsq;}
static void mark_direct_dispatch(struct scx_sched *sch, struct task_struct *ddsp_task, struct task_struct *p, u64 dsq_id, u64 enq_flags){ /* * Mark that dispatch already happened from ops.select_cpu() or * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value * which can never match a valid task pointer. */ __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
/* @p must match the task on the enqueue path */ if (unlikely(p != ddsp_task)) { if (IS_ERR(ddsp_task)) scx_error(sch, "%s[%d] already direct-dispatched", p->comm, p->pid); else scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]", ddsp_task->comm, ddsp_task->pid, p->comm, p->pid); return; }
WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); WARN_ON_ONCE(p->scx.ddsp_enq_flags);
p->scx.ddsp_dsq_id = dsq_id; p->scx.ddsp_enq_flags = enq_flags;}
/* * Clear @p direct dispatch state when leaving the scheduler. * * Direct dispatch state must be cleared in the following cases: * - direct_dispatch(): cleared on the synchronous enqueue path, deferred * dispatch keeps the state until consumed * - process_ddsp_deferred_locals(): cleared after consuming deferred state, * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch * verdict is ignored (local/global/bypass) * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred * cancellation and holding_cpu races * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by * the scx_bypass() loop, so that stale state is not reused by a subsequent * scheduler instance */static inline void clear_direct_dispatch(struct task_struct *p){ p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; p->scx.ddsp_enq_flags = 0;}
static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, u64 enq_flags){ struct rq *rq = task_rq(p); struct scx_dispatch_q *dsq = find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); u64 ddsp_enq_flags;
touch_core_sched_dispatch(rq, p);
p->scx.ddsp_enq_flags |= enq_flags;
/* * We are in the enqueue path with @rq locked and pinned, and thus can't * double lock a remote rq and enqueue to its local DSQ. For * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer * the enqueue so that it's executed when @rq can be unlocked. */ if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { unsigned long opss;
opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK;
switch (opss & SCX_OPSS_STATE_MASK) { case SCX_OPSS_NONE: break; case SCX_OPSS_QUEUEING: /* * As @p was never passed to the BPF side, _release is * not strictly necessary. Still do it for consistency. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); break; default: WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", p->comm, p->pid, opss); atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); break; }
WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); list_add_tail(&p->scx.dsq_list.node, &rq->scx.ddsp_deferred_locals); schedule_deferred_locked(rq); return; }
ddsp_enq_flags = p->scx.ddsp_enq_flags; clear_direct_dispatch(p);
dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);}
static bool scx_rq_online(struct rq *rq){ /* * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates * the online state as seen from the BPF scheduler. cpu_active() test * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will * stay set until the current scheduling operation is complete even if * we aren't locking @rq. */ return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));}
static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu){ struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; struct scx_dispatch_q *dsq; unsigned long qseq;
WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
/* internal movements - rq migration / RESTORE */ if (sticky_cpu == cpu_of(rq)) goto local_norefill;
/* * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr(). * Note that exiting and migration-disabled tasks that skip * ops.enqueue() below will lose IMMED protection unless * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set. */ p->scx.flags &= ~SCX_TASK_IMMED;
/* * If !scx_rq_online(), we already told the BPF scheduler that the CPU * is offline and are just running the hotplug path. Don't bother the * BPF scheduler. */ if (!scx_rq_online(rq)) goto local;
if (scx_bypassing(sch, cpu_of(rq))) { __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); goto bypass; }
if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct;
/* see %SCX_OPS_ENQ_EXITING */ if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && unlikely(p->flags & PF_EXITING)) { __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); goto local; }
/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && is_migration_disabled(p)) { __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); goto local; }
if (unlikely(!SCX_HAS_OP(sch, enqueue))) goto global;
/* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); WARN_ON_ONCE(*ddsp_taskp); *ddsp_taskp = p;
SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags);
*ddsp_taskp = NULL; if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct;
/* * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY * so ops.dequeue() is called when it leaves custody. */ p->scx.flags |= SCX_TASK_IN_CUSTODY;
/* * If not directly dispatched, QUEUEING isn't clear yet and dispatch or * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); return;
direct: direct_dispatch(sch, p, enq_flags); return;local_norefill: dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); return;local: dsq = &rq->scx.local_dsq; goto enqueue;global: dsq = find_global_dsq(sch, task_cpu(p)); goto enqueue;bypass: dsq = bypass_enq_target_dsq(sch, task_cpu(p)); goto enqueue;
enqueue: /* * For task-ordering, slice refill must be treated as implying the end * of the current slice. Otherwise, the longer @p stays on the CPU, the * higher priority it becomes from scx_prio_less()'s POV. */ touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); dispatch_enqueue(sch, rq, dsq, p, enq_flags);}
static bool task_runnable(const struct task_struct *p){ return !list_empty(&p->scx.runnable_node);}
static void set_task_runnable(struct rq *rq, struct task_struct *p){ lockdep_assert_rq_held(rq);
if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { p->scx.runnable_at = jiffies; p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; }
/* * list_add_tail() must be used. scx_bypass() depends on tasks being * appended to the runnable_list. */ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);}
static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at){ list_del_init(&p->scx.runnable_node); if (reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;}
static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags){ struct scx_sched *sch = scx_task_sched(p); int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags;
if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP;
/* * Restoring a running task will be immediately followed by * set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force * direct-dispatch into the local DSQ by setting the sticky_cpu. */ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) sticky_cpu = cpu_of(rq);
if (p->scx.flags & SCX_TASK_QUEUED) { WARN_ON_ONCE(!task_runnable(p)); goto out; }
set_task_runnable(rq, p); p->scx.flags |= SCX_TASK_QUEUED; rq->scx.nr_running++; add_nr_running(rq, 1);
if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags);
if (enq_flags & SCX_ENQ_WAKEUP) touch_core_sched(rq, p);
/* Start dl_server if this is the first task being enqueued */ if (rq->scx.nr_running == 1) dl_server_start(&rq->ext_server);
do_enqueue_task(rq, p, enq_flags, sticky_cpu);
if (sticky_cpu >= 0) p->scx.sticky_cpu = -1;out: rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
if ((enq_flags & SCX_ENQ_CPU_SELECTED) && unlikely(cpu_of(rq) != p->scx.selected_cpu)) __scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1);}
static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags){ struct scx_sched *sch = scx_task_sched(p); unsigned long opss;
/* dequeue is always temporary, don't reset runnable_at */ clr_task_runnable(p, false);
retry: /* acquire ensures that we see the preceding updates on QUEUED */ opss = atomic_long_read_acquire(&p->scx.ops_state);
switch (opss & SCX_OPSS_STATE_MASK) { case SCX_OPSS_NONE: break; case SCX_OPSS_QUEUEING: /* * QUEUEING is started and finished while holding @p's rq lock. * As we're holding the rq lock now, we shouldn't see QUEUEING. */ BUG(); case SCX_OPSS_QUEUED: /* * A queued task must always be in BPF scheduler's custody. If * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another * CPU has already passed call_task_dequeue() (which clears the * flag), but has not yet written SCX_OPSS_NONE. That final * store does not require this rq's lock, so retrying with * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, * handled by the fallthrough) on a subsequent iteration. */ if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) { cpu_relax(); goto retry; }
if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, SCX_OPSS_NONE)) break; fallthrough; case SCX_OPSS_DISPATCHING: /* * If @p is being dispatched from the BPF scheduler to a DSQ, * wait for the transfer to complete so that @p doesn't get * added to its DSQ after dequeueing is complete. * * As we're waiting on DISPATCHING with the rq locked, the * dispatching side shouldn't try to lock the rq while * DISPATCHING is set. See dispatch_to_local_dsq(). * * DISPATCHING shouldn't have qseq set and control can reach * here with NONE @opss from the above QUEUED case block. * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. */ wait_ops_state(p, SCX_OPSS_DISPATCHING); BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); break; }
/* * Call ops.dequeue() if the task is still in BPF custody. * * The code that clears ops_state to %SCX_OPSS_NONE does not always * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when * we're moving a task that was in %SCX_OPSS_DISPATCHING to a * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE * so that a concurrent dequeue can proceed, but we clear * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the * task. So we can see NONE + IN_CUSTODY here and we must handle * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until * it is enqueued on the destination. */ call_task_dequeue(sch, rq, p, deq_flags);}
static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags){ struct scx_sched *sch = scx_task_sched(p); u64 deq_flags = core_deq_flags;
/* * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property * change (not sleep or core-sched pick). */ if (!(deq_flags & (DEQUEUE_SLEEP | SCX_DEQ_CORE_SCHED_EXEC))) deq_flags |= SCX_DEQ_SCHED_CHANGE;
if (!(p->scx.flags & SCX_TASK_QUEUED)) { WARN_ON_ONCE(task_runnable(p)); return true; }
ops_dequeue(rq, p, deq_flags);
/* * A currently running task which is going off @rq first gets dequeued * and then stops running. As we want running <-> stopping transitions * to be contained within runnable <-> quiescent transitions, trigger * ->stopping() early here instead of in put_prev_task_scx(). * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while * balance_one() leaves @rq unlocked. However, they don't contain any * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) { update_curr_scx(rq); SCX_CALL_OP_TASK(sch, stopping, rq, p, false); }
if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags);
if (deq_flags & SCX_DEQ_SLEEP) p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; else p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP;
p->scx.flags &= ~SCX_TASK_QUEUED; rq->scx.nr_running--; sub_nr_running(rq, 1);
dispatch_dequeue(rq, p); clear_direct_dispatch(p); return true;}
static void yield_task_scx(struct rq *rq){ struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p);
if (SCX_HAS_OP(sch, yield)) SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL); else p->scx.slice = 0;}
static bool yield_to_task_scx(struct rq *rq, struct task_struct *to){ struct task_struct *from = rq->donor; struct scx_sched *sch = scx_task_sched(from);
if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to)) return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to); else return false;}
static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags){ /* * Preemption between SCX tasks is implemented by resetting the victim * task's slice to 0 and triggering reschedule on the target CPU. * Nothing to do. */ if (p->sched_class == &ext_sched_class) return;
/* * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. * This captures all preemption cases including: * * - A SCX task is currently running. * * - @rq is waking from idle due to a SCX task waking to it. * * - A higher-priority wakes up while SCX dispatch is in progress. */ if (rq->scx.nr_immed) schedule_reenq_local(rq, 0);}
static void move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *src_dsq, struct rq *dst_rq){ struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq;
/* @dsq is locked and @p is on @dst_rq */ lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq);
WARN_ON_ONCE(p->scx.holding_cpu >= 0);
if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) list_add(&p->scx.dsq_list.node, &dst_dsq->list); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
dsq_inc_nr(dst_dsq, p, enq_flags); p->scx.dsq = dst_dsq;
local_dsq_post_enq(sch, dst_dsq, p, enq_flags);}
/** * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ * @p: task to move * @enq_flags: %SCX_ENQ_* * @src_rq: rq to move the task from, locked on entry, released on return * @dst_rq: rq to move the task into, locked on return * * Move @p which is currently on @src_rq to @dst_rq's local DSQ. */static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq){ lockdep_assert_rq_held(src_rq);
/* * Set sticky_cpu before deactivate_task() to properly mark the * beginning of an SCX-internal migration. */ p->scx.sticky_cpu = cpu_of(dst_rq); deactivate_task(src_rq, p, 0); set_task_cpu(p, cpu_of(dst_rq));
switch_rq_lock(src_rq, dst_rq);
/* * We want to pass scx-specific enq_flags but activate_task() will * truncate the upper 32 bit. As we own @rq, we can pass them through * @rq->scx.extra_enq_flags instead. */ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); dst_rq->scx.extra_enq_flags = enq_flags; activate_task(dst_rq, p, 0); dst_rq->scx.extra_enq_flags = 0;}
/* * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two * differences: * * - is_cpu_allowed() asks "Can this task run on this CPU?" while * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to * this CPU?". * * While migration is disabled, is_cpu_allowed() has to say "yes" as the task * must be allowed to finish on the CPU that it's currently on regardless of * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the * BPF scheduler shouldn't attempt to migrate a task which has migration * disabled. * * - The BPF scheduler is bypassed while the rq is offline and we can always say * no to the BPF scheduler initiated migrations while offline. * * The caller must ensure that @p and @rq are on different CPUs. * If enforce == true, caller must hold @p's rq lock. */static bool task_can_run_on_remote_rq(struct scx_sched *sch, struct task_struct *p, struct rq *rq, bool enforce){ s32 cpu = cpu_of(rq);
/* * To prevent races with @p still running on its old CPU while switching * out, make sure we're holding @p's rq lock so as not to risk * erroneously killing the BPF scheduler. */ if (enforce) lockdep_assert_rq_held(task_rq(p));
WARN_ON_ONCE(task_cpu(p) == cpu);
/* * If @p has migration disabled, @p->cpus_ptr is updated to contain only * the pinned CPU in migrate_disable_switch() while @p is being switched * out. However, put_prev_task_scx() is called before @p->cpus_ptr is * updated and thus another CPU may see @p on a DSQ inbetween leading to * @p passing the below task_allowed_on_cpu() check while migration is * disabled. * * Test the migration disabled state first as the race window is narrow * and the BPF scheduler failing to check migration disabled state can * easily be masked if task_allowed_on_cpu() is done first. */ if (unlikely(is_migration_disabled(p))) { if (enforce) scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d", p->comm, p->pid, task_cpu(p), cpu); return false; }
/* * We don't require the BPF scheduler to avoid dispatching to offline * CPUs mostly for convenience but also because CPUs can go offline * between scx_bpf_dsq_insert() calls and here. Trigger error iff the * picked CPU is outside the allowed mask. */ if (!task_allowed_on_cpu(p, cpu)) { if (enforce) scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]", cpu, p->comm, p->pid); return false; }
if (!scx_rq_online(rq)) { if (enforce) __scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1); return false; }
return true;}
/** * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq * @p: target task * @dsq: locked DSQ @p is currently on * @src_rq: rq @p is currently on, stable with @dsq locked * * Called with @dsq locked but no rq's locked. We want to move @p to a different * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is * required when transferring into a local DSQ. Even when transferring into a * non-local DSQ, it's better to use the same mechanism to protect against * dequeues and maintain the invariant that @p->scx.dsq can only change while * @src_rq is locked, which e.g. scx_dump_task() depends on. * * We want to grab @src_rq but that can deadlock if we try while locking @dsq, * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As * this may race with dequeue, which can't drop the rq lock or fail, do a little * dancing from our side. * * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu * would be cleared to -1. While other cpus may have updated it to different * values afterwards, as this operation can't be preempted or recurse, the * holding_cpu can never become this CPU again before we're done. Thus, we can * tell whether we lost to dequeue by testing whether the holding_cpu still * points to this CPU. See dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue. */static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *src_rq){ s32 cpu = raw_smp_processor_id();
lockdep_assert_held(&dsq->lock);
WARN_ON_ONCE(p->scx.holding_cpu >= 0); task_unlink_from_dsq(p, dsq); p->scx.holding_cpu = cpu;
raw_spin_unlock(&dsq->lock); raw_spin_rq_lock(src_rq);
/* task_rq couldn't have changed if we're still the holding cpu */ return likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p));}
static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *dsq, struct rq *src_rq){ raw_spin_rq_unlock(this_rq);
if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); return true; } else { raw_spin_rq_unlock(src_rq); raw_spin_rq_lock(this_rq); return false; }}
/** * move_task_between_dsqs() - Move a task from one DSQ to another * @sch: scx_sched being operated on * @p: target task * @enq_flags: %SCX_ENQ_* * @src_dsq: DSQ @p is currently on, must not be a local DSQ * @dst_dsq: DSQ @p is being moved to, can be any DSQ * * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq * will change. As @p's task_rq is locked, this function doesn't need to use the * holding_cpu mechanism. * * On return, @src_dsq is unlocked and only @p's new task_rq, which is the * return value, is locked. */static struct rq *move_task_between_dsqs(struct scx_sched *sch, struct task_struct *p, u64 enq_flags, struct scx_dispatch_q *src_dsq, struct scx_dispatch_q *dst_dsq){ struct rq *src_rq = task_rq(p), *dst_rq;
BUG_ON(src_dsq->id == SCX_DSQ_LOCAL); lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(src_rq);
if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); if (src_rq != dst_rq && unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { dst_dsq = find_global_dsq(sch, task_cpu(p)); dst_rq = src_rq; enq_flags |= SCX_ENQ_GDSQ_FALLBACK; } } else { /* no need to migrate if destination is a non-local DSQ */ dst_rq = src_rq; }
/* * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different * CPU, @p will be migrated. */ if (dst_dsq->id == SCX_DSQ_LOCAL) { /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { task_unlink_from_dsq(p, src_dsq); move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); } else { raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(p, enq_flags, src_rq, dst_rq); } } else { /* * @p is going from a non-local DSQ to a non-local DSQ. As * $src_dsq is already locked, do an abbreviated dequeue. */ dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock);
dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); }
return dst_rq;}
static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags){ struct task_struct *p;retry: /* * The caller can't expect to successfully consume a task if the task's * addition to @dsq isn't guaranteed to be visible somehow. Test * @dsq->list without locking and skip if it seems empty. */ if (list_empty(&dsq->list)) return false;
raw_spin_lock(&dsq->lock);
nldsq_for_each_task(p, dsq) { struct rq *task_rq = task_rq(p);
/* * This loop can lead to multiple lockup scenarios, e.g. the BPF * scheduler can put an enormous number of affinitized tasks into * a contended DSQ, or the outer retry loop can repeatedly race * against scx_bypass() dequeueing tasks from @dsq trying to put * the system into the bypass mode. This can easily live-lock the * machine. If aborting, exit from all non-bypass DSQs. */ if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS) break;
if (rq == task_rq) { task_unlink_from_dsq(p, dsq); move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); return true; }
if (task_can_run_on_remote_rq(sch, p, rq, false)) { if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) return true; goto retry; } }
raw_spin_unlock(&dsq->lock); return false;}
static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq){ int node = cpu_to_node(cpu_of(rq));
return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);}
/** * dispatch_to_local_dsq - Dispatch a task to a local dsq * @sch: scx_sched being operated on * @rq: current rq which is locked * @dst_dsq: destination DSQ * @p: task to dispatch * @enq_flags: %SCX_ENQ_* * * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local * DSQ. This function performs all the synchronization dancing needed because * local DSQs are protected with rq locks. * * The caller must have exclusive ownership of @p (e.g. through * %SCX_OPSS_DISPATCHING). */static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dst_dsq, struct task_struct *p, u64 enq_flags){ struct rq *src_rq = task_rq(p); struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); struct rq *locked_rq = rq;
/* * We're synchronized against dequeue through DISPATCHING. As @p can't * be dequeued, its task_rq and cpus_allowed are stable too. * * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { dispatch_enqueue(sch, rq, dst_dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); return; }
/* * @p is on a possibly remote @src_rq which we need to lock to move the * task. If dequeue is in progress, it'd be locking @src_rq and waiting * on DISPATCHING, so we can't grab @src_rq lock while holding * DISPATCHING. * * As DISPATCHING guarantees that @p is wholly ours, we can pretend that * we're moving from a DSQ and use the same mechanism - mark the task * under transfer with holding_cpu, release DISPATCHING and then follow * the same protocol. See unlink_dsq_and_lock_src_rq(). */ p->scx.holding_cpu = raw_smp_processor_id();
/* store_release ensures that dequeue sees the above */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
/* switch to @src_rq lock */ if (locked_rq != src_rq) { switch_rq_lock(locked_rq, src_rq); locked_rq = src_rq; }
/* task_rq couldn't have changed if we're still the holding cpu */ if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by * abbreviating move_remote_task_to_local_dsq(). */ if (src_rq == dst_rq) { p->scx.holding_cpu = -1; dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); } else { move_remote_task_to_local_dsq(p, enq_flags, src_rq, dst_rq); /* task has been moved to dst_rq, which is now locked */ locked_rq = dst_rq; }
/* if the destination CPU is idle, wake it up */ if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class)) resched_curr(dst_rq); }
/* switch back to @rq lock */ if (locked_rq != rq) switch_rq_lock(locked_rq, rq);}
/** * finish_dispatch - Asynchronously finish dispatching a task * @rq: current rq which is locked * @p: task to finish dispatching * @qseq_at_dispatch: qseq when @p started getting dispatched * @dsq_id: destination DSQ ID * @enq_flags: %SCX_ENQ_* * * Dispatching to local DSQs may need to wait for queueing to complete or * require rq lock dancing. As we don't wanna do either while inside * ops.dispatch() to avoid locking order inversion, we split dispatching into * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the * task and its qseq. Once ops.dispatch() returns, this function is called to * finish up. * * There is no guarantee that @p is still valid for dispatching or even that it * was valid in the first place. Make sure that the task is still owned by the * BPF scheduler and claim the ownership before dispatching. */static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p, unsigned long qseq_at_dispatch, u64 dsq_id, u64 enq_flags){ struct scx_dispatch_q *dsq; unsigned long opss;
touch_core_sched_dispatch(rq, p);retry: /* * No need for _acquire here. @p is accessed only after a successful * try_cmpxchg to DISPATCHING. */ opss = atomic_long_read(&p->scx.ops_state);
switch (opss & SCX_OPSS_STATE_MASK) { case SCX_OPSS_DISPATCHING: case SCX_OPSS_NONE: /* someone else already got to it */ return; case SCX_OPSS_QUEUED: /* * If qseq doesn't match, @p has gone through at least one * dispatch/dequeue and re-enqueue cycle between * scx_bpf_dsq_insert() and here and we have no claim on it. */ if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) return;
/* see SCX_EV_INSERT_NOT_OWNED definition */ if (unlikely(!scx_task_on_sched(sch, p))) { __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); return; }
/* * While we know @p is accessible, we don't yet have a claim on * it - the BPF scheduler is allowed to dispatch tasks * spuriously and there can be a racing dequeue attempt. Let's * claim @p by atomically transitioning it from QUEUED to * DISPATCHING. */ if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, SCX_OPSS_DISPATCHING))) break; goto retry; case SCX_OPSS_QUEUEING: /* * do_enqueue_task() is in the process of transferring the task * to the BPF scheduler while holding @p's rq lock. As we aren't * holding any kernel or BPF resource that the enqueue path may * depend upon, it's safe to wait. */ wait_ops_state(p, opss); goto retry; }
BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED));
dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p));
if (dsq->id == SCX_DSQ_LOCAL) dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); else dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);}
static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq){ struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; u32 u;
for (u = 0; u < dspc->cursor; u++) { struct scx_dsp_buf_ent *ent = &dspc->buf[u];
finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id, ent->enq_flags); }
dspc->nr_tasks += dspc->cursor; dspc->cursor = 0;}
static inline void maybe_queue_balance_callback(struct rq *rq){ lockdep_assert_rq_held(rq);
if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) return;
queue_balance_callback(rq, &rq->scx.deferred_bal_cb, deferred_bal_cb_workfn);
rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;}
/* * One user of this function is scx_bpf_dispatch() which can be called * recursively as sub-sched dispatches nest. Always inline to reduce stack usage * from the call frame. */static __always_inline boolscx_dispatch_sched(struct scx_sched *sch, struct rq *rq, struct task_struct *prev, bool nested){ struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; int nr_loops = SCX_DSP_MAX_LOOPS; s32 cpu = cpu_of(rq); bool prev_on_sch = (prev->sched_class == &ext_sched_class) && scx_task_on_sched(sch, prev);
if (consume_global_dsq(sch, rq)) return true;
if (bypass_dsp_enabled(sch)) { /* if @sch is bypassing, only the bypass DSQs are active */ if (scx_bypassing(sch, cpu)) return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0);
#ifdef CONFIG_EXT_SUB_SCHED /* * If @sch isn't bypassing but its children are, @sch is * responsible for making forward progress for both its own * tasks that aren't bypassing and the bypassing descendants' * tasks. The following implements a simple built-in behavior - * let each CPU try to run the bypass DSQ every Nth time. * * Later, if necessary, we can add an ops flag to suppress the * auto-consumption and a kfunc to consume the bypass DSQ and, * so that the BPF scheduler can fully control scheduling of * bypassed tasks. */ struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return true; }#endif /* CONFIG_EXT_SUB_SCHED */ }
if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) return false;
dspc->rq = rq;
/* * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, * the local DSQ might still end up empty after a successful * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() * produced some tasks, retry. The BPF scheduler may depend on this * looping behavior to simplify its implementation. */ do { dspc->nr_tasks = 0;
if (nested) { SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), prev_on_sch ? prev : NULL); } else { /* stash @prev so that nested invocations can access it */ rq->scx.sub_dispatch_prev = prev; SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), prev_on_sch ? prev : NULL); rq->scx.sub_dispatch_prev = NULL; }
flush_dispatch_buf(sch, rq);
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { rq->scx.flags |= SCX_RQ_BAL_KEEP; return true; } if (rq->scx.local_dsq.nr) return true; if (consume_global_dsq(sch, rq)) return true;
/* * ops.dispatch() can trap us in this loop by repeatedly * dispatching ineligible tasks. Break out once in a while to * allow the watchdog to run. As IRQ can't be enabled in * balance(), we want to complete this scheduling cycle and then * start a new one. IOW, we want to call resched_curr() on the * next, most likely idle, task, not the current one. Use * __scx_bpf_kick_cpu() for deferred kicking. */ if (unlikely(!--nr_loops)) { scx_kick_cpu(sch, cpu, 0); break; } } while (dspc->nr_tasks);
/* * Prevent the CPU from going idle while bypassed descendants have tasks * queued. Without this fallback, bypassed tasks could stall if the host * scheduler's ops.dispatch() doesn't yield any tasks. */ if (bypass_dsp_enabled(sch)) return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0);
return false;}
static int balance_one(struct rq *rq, struct task_struct *prev){ struct scx_sched *sch = scx_root; s32 cpu = cpu_of(rq);
lockdep_assert_rq_held(rq); rq->scx.flags |= SCX_RQ_IN_BALANCE; rq->scx.flags &= ~SCX_RQ_BAL_KEEP;
if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) && unlikely(rq->scx.cpu_released)) { /* * If the previous sched_class for the current CPU was not SCX, * notify the BPF scheduler that it again has control of the * core. This callback complements ->cpu_release(), which is * emitted in switch_class(). */ if (sch->ops.cpu_acquire) SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL); rq->scx.cpu_released = false; }
if (prev->sched_class == &ext_sched_class) { update_curr_scx(rq);
/* * If @prev is runnable & has slice left, it has priority and * fetching more just increases latency for the fetched tasks. * Tell pick_task_scx() to keep running @prev. If the BPF * scheduler wants to handle this explicitly, it should * implement ->cpu_release(). * * See scx_disable_workfn() for the explanation on the bypassing * test. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && !scx_bypassing(sch, cpu)) { rq->scx.flags |= SCX_RQ_BAL_KEEP; goto has_tasks; } }
/* if there already are tasks to run, nothing to do */ if (rq->scx.local_dsq.nr) goto has_tasks;
if (scx_dispatch_sched(sch, rq, prev, false)) goto has_tasks;
/* * Didn't find another task to run. Keep running @prev unless * %SCX_OPS_ENQ_LAST is in effect. */ if ((prev->scx.flags & SCX_TASK_QUEUED) && (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) { rq->scx.flags |= SCX_RQ_BAL_KEEP; __scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1); goto has_tasks; } rq->scx.flags &= ~SCX_RQ_IN_BALANCE; return false;
has_tasks: /* * @rq may have extra IMMED tasks without reenq scheduled: * * - rq_is_open() can't reliably tell when and how slice is going to be * modified for $curr and allows IMMED tasks to be queued while * dispatch is in progress. * * - A non-IMMED HEAD task can get queued in front of an IMMED task * between the IMMED queueing and the subsequent scheduling event. */ if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) schedule_reenq_local(rq, 0);
rq->scx.flags &= ~SCX_RQ_IN_BALANCE; return true;}
static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first){ struct scx_sched *sch = scx_task_sched(p);
if (p->scx.flags & SCX_TASK_QUEUED) { /* * Core-sched might decide to execute @p before it is * dispatched. Call ops_dequeue() to notify the BPF scheduler. */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); dispatch_dequeue(rq, p); }
p->se.exec_start = rq_clock_task(rq);
/* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(sch, running, rq, p);
clr_task_runnable(p, true);
/* * @p is getting newly scheduled or got kicked after someone updated its * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be * stopped. See scx_can_stop_tick(). * * Moreover, refresh the load_avgs just when transitioning in and out of * nohz. In the future, we might want to add a mechanism to update * load_avgs periodically on tick-stopped CPUs. */ if (p->scx.slice == SCX_SLICE_INF) { if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { /* * Bypass mode always assigns finite slices, so @p * can't have an infinite slice while bypassing. * Therefore, sched_update_tick_dependency() can safely * evaluate the outgoing task. */ rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; sched_update_tick_dependency(rq);
update_other_load_avgs(rq); } } else { if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; update_other_load_avgs(rq); }
/* * @rq still references the outgoing scheduling context. A finite * slice is sufficient by itself to require the tick. */ if (tick_nohz_full_cpu(cpu_of(rq))) tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); }}
static enum scx_cpu_preempt_reasonpreempt_reason_from_class(const struct sched_class *class){ if (class == &stop_sched_class) return SCX_CPU_PREEMPT_STOP; if (class == &dl_sched_class) return SCX_CPU_PREEMPT_DL; if (class == &rt_sched_class) return SCX_CPU_PREEMPT_RT; return SCX_CPU_PREEMPT_UNKNOWN;}
static void switch_class(struct rq *rq, struct task_struct *next){ struct scx_sched *sch = scx_root; const struct sched_class *next_class = next->sched_class;
if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT)) return;
/* * The callback is conceptually meant to convey that the CPU is no * longer under the control of SCX. Therefore, don't invoke the callback * if the next class is below SCX (in which case the BPF scheduler has * actively decided not to schedule any tasks on the CPU). */ if (sched_class_above(&ext_sched_class, next_class)) return;
/* * At this point we know that SCX was preempted by a higher priority * sched_class, so invoke the ->cpu_release() callback if we have not * done so already. We only send the callback once between SCX being * preempted, and it regaining control of the CPU. * * ->cpu_release() complements ->cpu_acquire(), which is emitted the * next time that balance_one() is invoked. */ if (!rq->scx.cpu_released) { if (sch->ops.cpu_release) { struct scx_cpu_release_args args = { .reason = preempt_reason_from_class(next_class), .task = next, };
SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args); } rq->scx.cpu_released = true; }}
static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next){ struct scx_sched *sch = scx_task_sched(p);
/* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
update_curr_scx(rq);
/* see dequeue_task_scx() on why we skip when !QUEUED */ if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
if (p->scx.flags & SCX_TASK_QUEUED) { set_task_runnable(rq, p);
/* * If @p has slice left and is being put, @p is getting * preempted by a higher priority scheduler class or core-sched * forcing a different task. Leave it at the head of the local * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. */ if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); } goto switch_class; }
/* * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. * * Core scheduling can force this CPU idle while @p stays * runnable. @p's cookie then won't match the core's, so skip * the warning in that case. */ if (next && sched_class_above(&ext_sched_class, next->sched_class)) { WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { do_enqueue_task(rq, p, 0, -1); } }
switch_class: if (next && next->sched_class != &ext_sched_class) switch_class(rq, next);}
static void kick_sync_wait_bal_cb(struct rq *rq){ struct scx_kick_syncs __rcu *ks = __this_cpu_read(scx_kick_syncs); unsigned long *ksyncs = rcu_dereference_sched(ks)->syncs; bool waited; s32 cpu;
/* * Drop rq lock and enable IRQs while waiting. IRQs must be enabled * — a target CPU may be waiting for us to process an IPI (e.g. TLB * flush) while we wait for its kick_sync to advance. * * Also, keep advancing our own kick_sync so that new kick_sync waits * targeting us, which can start after we drop the lock, cannot form * cyclic dependencies. */retry: waited = false; for_each_cpu(cpu, rq->scx.cpus_to_sync) { /* * smp_load_acquire() pairs with smp_store_release() on * kick_sync updates on the target CPUs. */ if (cpu == cpu_of(rq) || smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) { cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync); continue; }
raw_spin_rq_unlock_irq(rq); while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) { smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); cpu_relax(); } raw_spin_rq_lock_irq(rq); waited = true; }
if (waited) goto retry;}
static struct task_struct *first_local_task(struct rq *rq){ return list_first_entry_or_null(&rq->scx.local_dsq.list, struct task_struct, scx.dsq_list.node);}
static struct task_struct *do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx){ struct task_struct *prev = rq->curr; bool keep_prev; struct task_struct *p;
/* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
rq_modified_begin(rq, &ext_sched_class);
rq_unpin_lock(rq, rf); balance_one(rq, prev); rq_repin_lock(rq, rf); maybe_queue_balance_callback(rq);
/* * Defer to a balance callback which can drop rq lock and enable * IRQs. Waiting directly in the pick path would deadlock against * CPUs sending us IPIs (e.g. TLB flushes) while we wait for them. */ if (unlikely(rq->scx.kick_sync_pending)) { rq->scx.kick_sync_pending = false; queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb); }
/* * If any higher-priority sched class enqueued a runnable task on * this rq during balance_one(), abort and return RETRY_TASK, so * that the scheduler loop can restart. * * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. */ if (!force_scx && rq_modified_above(rq, &ext_sched_class)) return RETRY_TASK;
keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP; if (unlikely(keep_prev && prev->sched_class != &ext_sched_class)) { WARN_ON_ONCE(scx_enable_state() == SCX_ENABLED); keep_prev = false; }
/* * If balance_one() is telling us to keep running @prev, replenish slice * if necessary and keep running @prev. Otherwise, pop the first one * from the local DSQ. */ if (keep_prev) { p = prev; if (!p->scx.slice) refill_task_slice_dfl(scx_task_sched(p), p); } else { p = first_local_task(rq); if (!p) return NULL;
if (unlikely(!p->scx.slice)) { struct scx_sched *sch = scx_task_sched(p);
if (!scx_bypassing(sch, cpu_of(rq)) && !sch->warned_zero_slice) { printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", p->comm, p->pid, __func__); sch->warned_zero_slice = true; } refill_task_slice_dfl(sch, p); } }
return p;}
static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf){ return do_pick_task_scx(rq, rf, false);}
/* * Select the next task to run from the ext scheduling class. * * Use do_pick_task_scx() directly with @force_scx enabled, since the * dl_server must always select a sched_ext task. */static struct task_struct *ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf){ if (!scx_enabled()) return NULL;
return do_pick_task_scx(dl_se->rq, rf, true);}
/* * Initialize the ext server deadline entity. */void ext_server_init(struct rq *rq){ struct sched_dl_entity *dl_se = &rq->ext_server;
init_dl_entity(dl_se);
dl_server_init(dl_se, rq, ext_server_pick_task);}
#ifdef CONFIG_SCHED_CORE/** * scx_prio_less - Task ordering for core-sched * @a: task A * @b: task B * @in_fi: in forced idle state * * Core-sched is implemented as an additional scheduling layer on top of the * usual sched_class'es and needs to find out the expected task ordering. For * SCX, core-sched calls this function to interrogate the task ordering. * * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used * to implement the default task ordering. The older the timestamp, the higher * priority the task - the global FIFO ordering matching the default scheduling * behavior. * * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to * implement FIFO ordering within each local DSQ. See pick_task_scx(). */bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, bool in_fi){ struct scx_sched *sch_a = scx_task_sched(a); struct scx_sched *sch_b = scx_task_sched(b);
/* * The const qualifiers are dropped from task_struct pointers when * calling ops.core_sched_before(). Accesses are controlled by the * verifier. */ if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) && !scx_bypassing(sch_a, task_cpu(a))) return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before, task_rq(a), (struct task_struct *)a, (struct task_struct *)b); else return time_after64(a->scx.core_sched_at, b->scx.core_sched_at);}#endif /* CONFIG_SCHED_CORE */
static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags){ struct scx_sched *sch = scx_task_sched(p); bool bypassing;
/* * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it * can be a good migration opportunity with low cache and memory * footprint. Returning a CPU different than @prev_cpu triggers * immediate rq migration. However, for SCX, as the current rq * association doesn't dictate where the task is going to run, this * doesn't fit well. If necessary, we can later add a dedicated method * which can decide to preempt self to force it through the regular * scheduling path. */ if (unlikely(wake_flags & WF_EXEC)) return prev_cpu;
bypassing = scx_bypassing(sch, task_cpu(p)); if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) { s32 cpu; struct task_struct **ddsp_taskp;
ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); WARN_ON_ONCE(*ddsp_taskp); *ddsp_taskp = p;
this_rq()->scx.in_select_cpu = true; cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, scx_cpu_arg(prev_cpu), wake_flags); cpu = scx_cpu_ret(sch, cpu); this_rq()->scx.in_select_cpu = false; p->scx.selected_cpu = cpu; *ddsp_taskp = NULL; if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()")) return cpu; else return prev_cpu; } else { s32 cpu;
cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0); if (cpu >= 0) { refill_task_slice_dfl(sch, p); p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; } else { cpu = prev_cpu; } p->scx.selected_cpu = cpu;
if (bypassing) __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); return cpu; }}
static void task_woken_scx(struct rq *rq, struct task_struct *p){ run_deferred(rq);}
static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ac){ struct scx_sched *sch = scx_task_sched(p);
set_cpus_allowed_common(p, ac);
if (task_dead_and_done(p)) return;
/* * The effective cpumask is stored in @p->cpus_ptr which may temporarily * differ from the configured one in @p->cpus_mask. Always tell the bpf * scheduler the effective one. * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. */ if (SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);}
static void handle_hotplug(struct rq *rq, bool online){ struct scx_sched *sch = scx_root; s32 cpu = cpu_of(rq);
atomic_long_inc(&scx_hotplug_seq);
/* * scx_root updates are protected by cpus_read_lock() and will stay * stable here. Note that we can't depend on scx_enabled() test as the * hotplug ops need to be enabled before __scx_enabled is set. */ if (unlikely(!sch)) return;
if (scx_enabled()) scx_idle_update_selcpu_topology(&sch->ops);
if (online && SCX_HAS_OP(sch, cpu_online)) SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu)); else if (!online && SCX_HAS_OP(sch, cpu_offline)) SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu)); else scx_exit(sch, SCX_EXIT_UNREG_KERN, SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, "cpu %d going %s, exiting scheduler", cpu, online ? "online" : "offline");}
void scx_rq_activate(struct rq *rq){ handle_hotplug(rq, true);}
void scx_rq_deactivate(struct rq *rq){ handle_hotplug(rq, false);}
static void rq_online_scx(struct rq *rq){ rq->scx.flags |= SCX_RQ_ONLINE;}
static void rq_offline_scx(struct rq *rq){ rq->scx.flags &= ~SCX_RQ_ONLINE;}
static bool check_rq_for_timeouts(struct rq *rq){ struct scx_sched *sch; struct task_struct *p; struct rq_flags rf; bool timed_out = false;
rq_lock_irqsave(rq, &rf); sch = rcu_dereference_bh(scx_root); if (unlikely(!sch)) goto out_unlock;
list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { struct scx_sched *sch = scx_task_sched(p); unsigned long last_runnable = p->scx.runnable_at;
if (unlikely(time_after(jiffies, last_runnable + READ_ONCE(sch->watchdog_timeout)))) { u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
__scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), "%s[%d] failed to run for %u.%03us", p->comm, p->pid, dur_ms / 1000, dur_ms % 1000); timed_out = true; break; } }out_unlock: rq_unlock_irqrestore(rq, &rf); return timed_out;}
static void scx_watchdog_workfn(struct work_struct *work){ unsigned long intv; int cpu;
WRITE_ONCE(scx_watchdog_timestamp, jiffies);
for_each_online_cpu(cpu) { if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) break;
cond_resched(); }
intv = READ_ONCE(scx_watchdog_interval); if (intv < ULONG_MAX) queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv);}
void scx_tick(struct rq *rq){ struct scx_sched *root; unsigned long last_check;
if (!scx_enabled()) return;
root = rcu_dereference_bh(scx_root); if (unlikely(!root)) return;
last_check = READ_ONCE(scx_watchdog_timestamp); if (unlikely(time_after(jiffies, last_check + READ_ONCE(root->watchdog_timeout)))) { u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
scx_exit(root, SCX_EXIT_ERROR_STALL, 0, "watchdog failed to check in for %u.%03us", dur_ms / 1000, dur_ms % 1000); }
update_other_load_avgs(rq);}
static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued){ struct scx_sched *sch = scx_task_sched(curr);
update_curr_scx(rq);
/* * While disabling, always resched and refresh core-sched timestamp as * we can't trust the slice management or ops.core_sched_before(). */ if (scx_bypassing(sch, cpu_of(rq))) { curr->scx.slice = 0; touch_core_sched(rq, curr); } else if (SCX_HAS_OP(sch, tick)) { SCX_CALL_OP_TASK(sch, tick, rq, curr); }
if (!curr->scx.slice) resched_curr(rq);}
#ifdef CONFIG_EXT_GROUP_SCHEDstatic struct cgroup *tg_cgrp(struct task_group *tg){ /* * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the * root cgroup. */ if (tg && tg->css.cgroup) return tg->css.cgroup; else return &cgrp_dfl_root.cgrp;}
#define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg),
#else /* CONFIG_EXT_GROUP_SCHED */
#define SCX_INIT_TASK_ARGS_CGROUP(tg)
#endif /* CONFIG_EXT_GROUP_SCHED */
static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork){ int ret;
p->scx.disallow = false;
if (SCX_HAS_OP(sch, init_task)) { struct scx_init_task_args args = { SCX_INIT_TASK_ARGS_CGROUP(task_group(p)) .fork = fork, };
ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); if (unlikely(ret)) { ret = ops_sanitize_err(sch, "init_task", ret); return ret; } }
if (p->scx.disallow) { if (unlikely(scx_parent(sch))) { scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]", p->comm, p->pid); } else if (unlikely(fork)) { scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", p->comm, p->pid); } else { struct rq *rq; struct rq_flags rf;
rq = task_rq_lock(p, &rf);
/* * We're in the load path and @p->policy will be applied * right after. Reverting @p->policy here and rejecting * %SCHED_EXT transitions from scx_check_setscheduler() * guarantees that if ops.init_task() sets @p->disallow, * @p can never be in SCX. */ if (p->policy == SCHED_EXT) { p->policy = SCHED_NORMAL; atomic_long_inc(&scx_nr_rejected); }
task_rq_unlock(rq, p, &rf); } }
return 0;}
static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p){ struct rq *rq = task_rq(p); u32 weight;
lockdep_assert_rq_held(rq);
/* * Verify the task is not in BPF scheduler's custody. If flag * transitions are consistent, the flag should always be clear * here. */ WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
/* * Set the weight before calling ops.enable() so that the scheduler * doesn't see a stale value if they inspect the task struct. */ if (task_has_idle_policy(p)) weight = WEIGHT_IDLEPRIO; else weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
p->scx.weight = sched_weight_to_cgroup(weight);
if (SCX_HAS_OP(sch, enable)) SCX_CALL_OP_TASK(sch, enable, rq, p);
if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);}
static void scx_enable_task(struct scx_sched *sch, struct task_struct *p){ __scx_enable_task(sch, p); scx_set_task_state(p, SCX_TASK_ENABLED);}
static void scx_disable_task(struct scx_sched *sch, struct task_struct *p){ struct rq *rq = task_rq(p);
lockdep_assert_rq_held(rq); WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
clear_direct_dispatch(p);
if (SCX_HAS_OP(sch, disable)) SCX_CALL_OP_TASK(sch, disable, rq, p); scx_set_task_state(p, SCX_TASK_READY);
/* * Reset the SCX-managed fields when @p leaves the BPF scheduler's * control, after ops.disable() has observed their final values. */ p->scx.dsq_vtime = 0; p->scx.slice = 0;
/* * Verify the task is not in BPF scheduler's custody. If flag * transitions are consistent, the flag should always be clear * here. */ WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);}
static void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p){ struct scx_exit_task_args args = { .cancelled = false, };
lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p));
switch (scx_get_task_state(p)) { case SCX_TASK_NONE: return; case SCX_TASK_INIT: args.cancelled = true; break; case SCX_TASK_READY: break; case SCX_TASK_ENABLED: scx_disable_task(sch, p); break; default: WARN_ON_ONCE(true); return; }
if (SCX_HAS_OP(sch, exit_task)) SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);}
/* * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never * ran. The task state has not been transitioned, so this mirrors the * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). */static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p){ struct scx_exit_task_args args = { .cancelled = true };
lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p));
if (SCX_HAS_OP(sch, exit_task)) SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);}
static void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p){ __scx_disable_and_exit_task(sch, p);
/* * If set, @p exited between __scx_init_task() and scx_enable_task() in * scx_sub_enable() and is initialized for both the associated sched and * its parent. Exit for the child too - scx_enable_task() never ran for * it, so undo only init_task. The flag is only set on the sub-enable * path, so it's always clear when @p arrives here in %SCX_TASK_NONE. */ if (p->scx.flags & SCX_TASK_SUB_INIT) { if (!WARN_ON_ONCE(!scx_enabling_sub_sched)) scx_sub_init_cancel_task(scx_enabling_sub_sched, p); p->scx.flags &= ~SCX_TASK_SUB_INIT; }
scx_set_task_sched(p, NULL); scx_set_task_state(p, SCX_TASK_NONE);}
void init_scx_entity(struct sched_ext_entity *scx){ memset(scx, 0, sizeof(*scx)); INIT_LIST_HEAD(&scx->dsq_list.node); RB_CLEAR_NODE(&scx->dsq_priq); scx->sticky_cpu = -1; scx->holding_cpu = -1; INIT_LIST_HEAD(&scx->runnable_node); scx->runnable_at = jiffies; scx->ddsp_dsq_id = SCX_DSQ_INVALID; scx->slice = SCX_SLICE_DFL;}
/* See scx_tid_alloc / scx_tid_cursor. */static u64 scx_alloc_tid(void){ struct scx_tid_alloc *ta;
guard(preempt)(); ta = this_cpu_ptr(&scx_tid_alloc);
if (unlikely(ta->next >= ta->end)) { ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor); ta->end = ta->next + SCX_TID_CHUNK; } return ta->next++;}
static void scx_tid_hash_insert(struct task_struct *p){ int ret;
lockdep_assert_held(&scx_tasks_lock);
ret = rhashtable_lookup_insert_fast(&scx_tid_hash, &p->scx.tid_hash_node, scx_tid_hash_params); WARN_ON_ONCE(ret);}
void scx_pre_fork(struct task_struct *p){ /* * BPF scheduler enable/disable paths want to be able to iterate and * update all tasks which can become complex when racing forks. As * enable/disable are very cold paths, let's use a percpu_rwsem to * exclude forks. */ percpu_down_read(&scx_fork_rwsem);}
int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs){ s32 ret;
percpu_rwsem_assert_held(&scx_fork_rwsem);
p->scx.tid = scx_alloc_tid();
if (scx_init_task_enabled) {#ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched;#else struct scx_sched *sch = scx_root;#endif scx_set_task_state(p, SCX_TASK_INIT_BEGIN); ret = __scx_init_task(sch, p, true); if (unlikely(ret)) { scx_set_task_state(p, SCX_TASK_NONE); return ret; } scx_set_task_state(p, SCX_TASK_INIT); scx_set_task_sched(p, sch); }
return 0;}
void scx_post_fork(struct task_struct *p){ if (scx_init_task_enabled) { scx_set_task_state(p, SCX_TASK_READY);
/* * Enable the task immediately if it's running on sched_ext. * Otherwise, it'll be enabled in switching_to_scx() if and * when it's ever configured to run with a SCHED_EXT policy. */ if (p->sched_class == &ext_sched_class) { struct rq_flags rf; struct rq *rq;
rq = task_rq_lock(p, &rf); scx_enable_task(scx_task_sched(p), p); task_rq_unlock(rq, p, &rf); } }
scoped_guard(raw_spinlock_irq, &scx_tasks_lock) { list_add_tail(&p->scx.tasks_node, &scx_tasks); if (scx_tid_to_task_enabled()) scx_tid_hash_insert(p); }
percpu_up_read(&scx_fork_rwsem);}
void scx_cancel_fork(struct task_struct *p){ if (scx_enabled()) { struct rq *rq; struct rq_flags rf;
rq = task_rq_lock(p, &rf); WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); scx_disable_and_exit_task(scx_task_sched(p), p); task_rq_unlock(rq, p, &rf); }
percpu_up_read(&scx_fork_rwsem);}
/** * task_dead_and_done - Is a task dead and done running? * @p: target task * * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the * task no longer exists from SCX's POV. However, certain sched_class ops may be * invoked on these dead tasks leading to failures - e.g. sched_setscheduler() * may try to switch a task which finished sched_ext_dead() back into SCX * triggering invalid SCX task state transitions and worse. * * Once a task has finished the final switch, sched_ext_dead() is the only thing * that needs to happen on the task. Use this test to short-circuit sched_class * operations which may be called on dead tasks. */static bool task_dead_and_done(struct task_struct *p){ struct rq *rq = task_rq(p);
lockdep_assert_rq_held(rq);
/* * In do_task_dead(), a dying task sets %TASK_DEAD with preemption * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p * won't ever run again. */ return unlikely(READ_ONCE(p->__state) == TASK_DEAD) && !task_on_cpu(rq, p);}
void sched_ext_dead(struct task_struct *p){ /* * By the time control reaches here, @p has %TASK_DEAD set, switched out * for the last time and then dropped the rq lock - task_dead_and_done() * should be returning %true nullifying the straggling sched_class ops. * Remove from scx_tasks and exit @p. */ scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) { list_del_init(&p->scx.tasks_node); if (scx_tid_to_task_enabled()) rhashtable_remove_fast(&scx_tid_hash, &p->scx.tid_hash_node, scx_tid_hash_params); }
/* * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> * ENABLED transitions can't race us. Disable ops for @p. * * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup * iteration is only used from sub-sched paths, which require root * enabled. Root enable transitions every live task to at least READY. * * %INIT_BEGIN means ops.init_task() is running for @p. Don't call * into ops; transition to %DEAD so the post-init recheck unwinds * via scx_sub_init_cancel_task(). */ if (scx_get_task_state(p) != SCX_TASK_NONE) { struct rq_flags rf; struct rq *rq;
rq = task_rq_lock(p, &rf); if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN) scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_state(p, SCX_TASK_DEAD); task_rq_unlock(rq, p, &rf); }}
static void reweight_task_scx(struct rq *rq, struct task_struct *p, const struct load_weight *lw){ struct scx_sched *sch = scx_task_sched(p);
lockdep_assert_rq_held(task_rq(p));
if (task_dead_and_done(p)) return;
/* * When switching sched_class away from SCX, reweight_task_scx() * is called _after_ scx_disable_task(). Skip calling ops.set_weight() * since the BPF scheduler may have already forgotten the task in * ops.disable(). * p->scx.weight will be recalculated in scx_enable_task() if the task * ever returns to SCX class. */ if (scx_get_task_state(p) != SCX_TASK_ENABLED) return;
p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);}
static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio){}
static void switching_to_scx(struct rq *rq, struct task_struct *p){ struct scx_sched *sch = scx_task_sched(p);
if (task_dead_and_done(p)) return;
scx_enable_task(sch, p);
/* * set_cpus_allowed_scx() is not called while @p is associated with a * different scheduler class. Keep the BPF scheduler up-to-date. */ if (SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);}
static void switched_from_scx(struct rq *rq, struct task_struct *p){ if (task_dead_and_done(p)) return;
/* * %NONE means SCX is no longer tracking @p at the task level (e.g. * scx_fail_parent() handed @p back to the parent at NONE pending the * parent's own teardown). There is nothing to disable; calling * scx_disable_task() would WARN on the non-%ENABLED state and trigger a * NONE -> READY validation failure. */ if (scx_get_task_state(p) == SCX_TASK_NONE) return;
scx_disable_task(scx_task_sched(p), p);}
static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
int scx_check_setscheduler(struct task_struct *p, int policy){ lockdep_assert_rq_held(task_rq(p));
/* if disallow, reject transitioning into SCX */ if (scx_enabled() && READ_ONCE(p->scx.disallow) && p->policy != policy && policy == SCHED_EXT) return -EACCES;
return 0;}
static void process_ddsp_deferred_locals(struct rq *rq){ struct task_struct *p;
lockdep_assert_rq_held(rq);
/* * Now that @rq can be unlocked, execute the deferred enqueueing of * tasks directly dispatched to the local DSQs of other CPUs. See * direct_dispatch(). Keep popping from the head instead of using * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq * temporarily. */ while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, struct task_struct, scx.dsq_list.node))) { struct scx_sched *sch = scx_task_sched(p); struct scx_dispatch_q *dsq; u64 dsq_id = p->scx.ddsp_dsq_id; u64 enq_flags = p->scx.ddsp_enq_flags;
list_del_init(&p->scx.dsq_list.node); clear_direct_dispatch(p);
dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p)); if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); }}
/* * Determine whether @p should be reenqueued from a local DSQ. * * @reenq_flags is mutable and accumulates state across the DSQ walk: * * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First" * tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at * the head consumes the first slot. * * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if * rq_is_open() is true. * * An IMMED task is kept (returns %false) only if it's the first task in the DSQ * AND the current task is done — i.e. it will execute immediately. All other * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head, * every IMMED task behind it gets reenqueued. * * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ | * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT * in process_deferred_reenq_locals(). */static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason){ bool first;
first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST;
*reason = SCX_TASK_REENQ_KFUNC;
if ((p->scx.flags & SCX_TASK_IMMED) && (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) { __scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1); *reason = SCX_TASK_REENQ_IMMED; return true; }
return *reenq_flags & SCX_REENQ_ANY;}
static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags){ LIST_HEAD(tasks); u32 nr_enqueued = 0; struct task_struct *p, *n;
lockdep_assert_rq_held(rq);
if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK)) reenq_flags &= ~__SCX_REENQ_TSR_MASK; if (rq_is_open(rq, 0)) reenq_flags |= SCX_REENQ_TSR_RQ_OPEN;
/* * The BPF scheduler may choose to dispatch tasks back to * @rq->scx.local_dsq. Move all candidate tasks off to a private list * first to avoid processing the same tasks repeatedly. */ list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, scx.dsq_list.node) { struct scx_sched *task_sch = scx_task_sched(p); u32 reason;
/* * If @p is being migrated, @p's current CPU may not agree with * its allowed CPUs and the migration_cpu_stop is about to * deactivate and re-activate @p anyway. Skip re-enqueueing. * * While racing sched property changes may also dequeue and * re-enqueue a migrating task while its current CPU and allowed * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to * the current local DSQ for running tasks and thus are not * visible to the BPF scheduler. */ if (p->migration_pending) continue;
if (!scx_is_descendant(task_sch, sch)) continue;
if (!local_task_should_reenq(p, &reenq_flags, &reason)) continue;
dispatch_dequeue(rq, p);
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; p->scx.flags |= reason;
list_add_tail(&p->scx.dsq_list.node, &tasks); }
list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { list_del_init(&p->scx.dsq_list.node);
do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; }
return nr_enqueued;}
static void process_deferred_reenq_locals(struct rq *rq){ u64 seq = ++rq->scx.deferred_reenq_locals_seq;
lockdep_assert_rq_held(rq);
while (true) { struct scx_sched *sch; u64 reenq_flags; bool skip = false;
scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_local *drl = list_first_entry_or_null(&rq->scx.deferred_reenq_locals, struct scx_deferred_reenq_local, node); struct scx_sched_pcpu *sch_pcpu;
if (!drl) return;
sch_pcpu = container_of(drl, struct scx_sched_pcpu, deferred_reenq_local); sch = sch_pcpu->sch;
reenq_flags = drl->flags; WRITE_ONCE(drl->flags, 0); list_del_init(&drl->node);
if (likely(drl->seq != seq)) { drl->seq = seq; drl->cnt = 0; } else { if (unlikely(++drl->cnt > SCX_REENQ_LOCAL_MAX_REPEAT)) { scx_error(sch, "SCX_ENQ_REENQ on SCX_DSQ_LOCAL repeated %u times", drl->cnt); skip = true; }
__scx_add_event(sch, SCX_EV_REENQ_LOCAL_REPEAT, 1); } }
if (!skip) { /* see schedule_dsq_reenq() */ smp_mb();
reenq_local(sch, rq, reenq_flags); } }}
static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason){ *reason = SCX_TASK_REENQ_KFUNC; return reenq_flags & SCX_REENQ_ANY;}
static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags){ struct rq *locked_rq = rq; struct scx_sched *sch = dsq->sched; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); struct task_struct *p; s32 nr_enqueued = 0;
lockdep_assert_rq_held(rq);
raw_spin_lock(&dsq->lock);
while (likely(!READ_ONCE(sch->bypass_depth))) { struct rq *task_rq; u32 reason;
p = nldsq_cursor_next_task(&cursor, dsq); if (!p) break;
if (!user_task_should_reenq(p, reenq_flags, &reason)) continue;
task_rq = task_rq(p);
if (locked_rq != task_rq) { if (locked_rq) raw_spin_rq_unlock(locked_rq); if (unlikely(!raw_spin_rq_trylock(task_rq))) { raw_spin_unlock(&dsq->lock); raw_spin_rq_lock(task_rq); raw_spin_lock(&dsq->lock); } locked_rq = task_rq;
/* did we lose @p while switching locks? */ if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p)) continue; }
/* @p is on @dsq, its rq and @dsq are locked */ dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock);
if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; p->scx.flags |= reason;
do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { raw_spin_rq_unlock(locked_rq); locked_rq = NULL; cpu_relax(); }
raw_spin_lock(&dsq->lock); }
list_del_init(&cursor.node); raw_spin_unlock(&dsq->lock);
if (locked_rq != rq) { if (locked_rq) raw_spin_rq_unlock(locked_rq); raw_spin_rq_lock(rq); }}
static void process_deferred_reenq_users(struct rq *rq){ lockdep_assert_rq_held(rq);
while (true) { struct scx_dispatch_q *dsq; u64 reenq_flags;
scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) { struct scx_deferred_reenq_user *dru = list_first_entry_or_null(&rq->scx.deferred_reenq_users, struct scx_deferred_reenq_user, node); struct scx_dsq_pcpu *dsq_pcpu;
if (!dru) return;
dsq_pcpu = container_of(dru, struct scx_dsq_pcpu, deferred_reenq_user); dsq = dsq_pcpu->dsq; reenq_flags = dru->flags; WRITE_ONCE(dru->flags, 0); list_del_init(&dru->node); }
/* see schedule_dsq_reenq() */ smp_mb();
BUG_ON(dsq->id & SCX_DSQ_FLAG_BUILTIN); reenq_user(rq, dsq, reenq_flags); }}
static void run_deferred(struct rq *rq){ process_ddsp_deferred_locals(rq);
if (!list_empty(&rq->scx.deferred_reenq_locals)) process_deferred_reenq_locals(rq);
if (!list_empty(&rq->scx.deferred_reenq_users)) process_deferred_reenq_users(rq);}
#ifdef CONFIG_NO_HZ_FULLbool scx_can_stop_tick(struct rq *rq){ struct task_struct *p = rq->curr; struct scx_sched *sch = scx_task_sched(p);
if (p->sched_class != &ext_sched_class) return true;
/* * @rq->curr may still reference an outgoing EXT task after it has been * dequeued. If no EXT tasks are accounted on @rq, ignore its stale * slice state. If another task is dispatched from a DSQ, * set_next_task_scx() will update the dependency for the incoming task. */ if (!rq->scx.nr_running) return true;
if (scx_bypassing(sch, cpu_of(rq))) return false;
/* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks * iff the BPF scheduler indicated so. See set_next_task_scx(). */ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;}#endif
#ifdef CONFIG_EXT_GROUP_SCHED
DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem);static bool scx_cgroup_enabled;
void scx_tg_init(struct task_group *tg){ tg->scx.weight = CGROUP_WEIGHT_DFL; tg->scx.bw_period_us = default_bw_period_us(); tg->scx.bw_quota_us = RUNTIME_INF; tg->scx.idle = false;}
int scx_tg_online(struct task_group *tg){ struct scx_sched *sch = scx_root; int ret = 0;
WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED));
if (scx_cgroup_enabled) { if (SCX_HAS_OP(sch, cgroup_init)) { struct scx_cgroup_init_args args = { .weight = tg->scx.weight, .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us };
ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); if (ret) ret = ops_sanitize_err(sch, "cgroup_init", ret); } if (ret == 0) tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; } else { tg->scx.flags |= SCX_TG_ONLINE; }
return ret;}
void scx_tg_offline(struct task_group *tg){ struct scx_sched *sch = scx_root;
WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE));
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_exit) && (tg->scx.flags & SCX_TG_INITED)) SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup); tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED);}
int scx_cgroup_can_attach(struct cgroup_taskset *tset){ struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p; int ret;
if (!scx_cgroup_enabled) return 0;
cgroup_taskset_for_each(p, css, tset) { struct cgroup *from = tg_cgrp(task_group(p)); struct cgroup *to = tg_cgrp(css_tg(css));
WARN_ON_ONCE(p->scx.cgrp_moving_from);
/* * sched_move_task() omits identity migrations. Let's match the * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() * always match one-to-one. */ if (from == to) continue;
if (SCX_HAS_OP(sch, cgroup_prep_move)) { ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL, p, from, css->cgroup); if (ret) goto err; }
p->scx.cgrp_moving_from = from; }
return 0;
err: cgroup_taskset_for_each(p, css, tset) { if (SCX_HAS_OP(sch, cgroup_cancel_move) && p->scx.cgrp_moving_from) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; }
return ops_sanitize_err(sch, "cgroup_prep_move", ret);}
void scx_cgroup_move_task(struct task_struct *p){ struct scx_sched *sch = scx_root;
if (!scx_cgroup_enabled) return;
/* * scx_cgroup_can_attach() sets cgrp_moving_from only when the task's * cgroup changes. Migration keys off css rather than cgroup identity, * so it can hand an unchanged-cgroup task here with cgrp_moving_from * NULL. Nothing to report to the BPF scheduler then, so skip it and * keep prep_move and move paired. */ if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from) SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p), p, p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); p->scx.cgrp_moving_from = NULL;}
void scx_cgroup_cancel_attach(struct cgroup_taskset *tset){ struct scx_sched *sch = scx_root; struct cgroup_subsys_state *css; struct task_struct *p;
if (!scx_cgroup_enabled) return;
cgroup_taskset_for_each(p, css, tset) { if (SCX_HAS_OP(sch, cgroup_cancel_move) && p->scx.cgrp_moving_from) SCX_CALL_OP(sch, cgroup_cancel_move, NULL, p, p->scx.cgrp_moving_from, css->cgroup); p->scx.cgrp_moving_from = NULL; }}
void scx_group_set_weight(struct task_group *tg, unsigned long weight){ struct scx_sched *sch;
percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_root;
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight);
tg->scx.weight = weight;
percpu_up_read(&scx_cgroup_ops_rwsem);}
void scx_group_set_idle(struct task_group *tg, bool idle){ struct scx_sched *sch;
percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_root;
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
/* Update the task group's idle state */ tg->scx.idle = idle;
percpu_up_read(&scx_cgroup_ops_rwsem);}
void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us){ struct scx_sched *sch;
percpu_down_read(&scx_cgroup_ops_rwsem); sch = scx_root;
if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || tg->scx.bw_quota_us != quota_us || tg->scx.bw_burst_us != burst_us)) SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL, tg_cgrp(tg), period_us, quota_us, burst_us);
tg->scx.bw_period_us = period_us; tg->scx.bw_quota_us = quota_us; tg->scx.bw_burst_us = burst_us;
percpu_up_read(&scx_cgroup_ops_rwsem);}#endif /* CONFIG_EXT_GROUP_SCHED */
#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)static struct cgroup *root_cgroup(void){ return &cgrp_dfl_root.cgrp;}
static void scx_cgroup_lock(void){#ifdef CONFIG_EXT_GROUP_SCHED percpu_down_write(&scx_cgroup_ops_rwsem);#endif cgroup_lock();}
static void scx_cgroup_unlock(void){ cgroup_unlock();#ifdef CONFIG_EXT_GROUP_SCHED percpu_up_write(&scx_cgroup_ops_rwsem);#endif}#else /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */static inline struct cgroup *root_cgroup(void) { return NULL; }static inline void scx_cgroup_lock(void) {}static inline void scx_cgroup_unlock(void) {}#endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
#ifdef CONFIG_EXT_SUB_SCHEDstatic struct cgroup *sch_cgroup(struct scx_sched *sch){ return sch->cgrp;}
/* for each descendant of @cgrp including self, set ->scx_sched to @sch */static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch){ struct cgroup *pos; struct cgroup_subsys_state *css;
cgroup_for_each_live_descendant_pre(pos, css, cgrp) rcu_assign_pointer(pos->scx_sched, sch);}#else /* CONFIG_EXT_SUB_SCHED */static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}#endif /* CONFIG_EXT_SUB_SCHED */
/* * Omitted operations: * * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. * * - task_fork/dead: We need fork/dead notifications for all tasks regardless of * their current sched_class. Call them directly from sched core instead. */DEFINE_SCHED_CLASS(ext) = { .enqueue_task = enqueue_task_scx, .dequeue_task = dequeue_task_scx, .yield_task = yield_task_scx, .yield_to_task = yield_to_task_scx,
.wakeup_preempt = wakeup_preempt_scx,
.pick_task = pick_task_scx,
.put_prev_task = put_prev_task_scx, .set_next_task = set_next_task_scx,
.select_task_rq = select_task_rq_scx, .task_woken = task_woken_scx, .set_cpus_allowed = set_cpus_allowed_scx,
.rq_online = rq_online_scx, .rq_offline = rq_offline_scx,
.task_tick = task_tick_scx,
.switching_to = switching_to_scx, .switched_from = switched_from_scx, .switched_to = switched_to_scx, .reweight_task = reweight_task_scx, .prio_changed = prio_changed_scx,
.update_curr = update_curr_scx,
#ifdef CONFIG_UCLAMP_TASK .uclamp_enabled = 1,#endif};
static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch){ s32 cpu;
memset(dsq, 0, sizeof(*dsq));
raw_spin_lock_init(&dsq->lock); INIT_LIST_HEAD(&dsq->list); dsq->id = dsq_id; dsq->sched = sch;
dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); if (!dsq->pcpu) return -ENOMEM;
for_each_possible_cpu(cpu) { struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
pcpu->dsq = dsq; INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); }
return 0;}
static void exit_dsq(struct scx_dispatch_q *dsq){ s32 cpu;
for_each_possible_cpu(cpu) { struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; struct rq *rq = cpu_rq(cpu);
/* * There must have been a RCU grace period since the last * insertion and @dsq should be off the deferred list by now. */ if (WARN_ON_ONCE(!list_empty(&dru->node))) { guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock); list_del_init(&dru->node); } }
free_percpu(dsq->pcpu);}
static void free_dsq_rcufn(struct rcu_head *rcu){ struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu);
exit_dsq(dsq); kfree(dsq);}
static void free_dsq_irq_workfn(struct irq_work *irq_work){ struct llist_node *to_free = llist_del_all(&dsqs_to_free); struct scx_dispatch_q *dsq, *tmp_dsq;
llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) call_rcu(&dsq->rcu, free_dsq_rcufn);}
static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
static void destroy_dsq(struct scx_sched *sch, u64 dsq_id){ struct scx_dispatch_q *dsq; unsigned long flags;
rcu_read_lock();
dsq = find_user_dsq(sch, dsq_id); if (!dsq) goto out_unlock_rcu;
raw_spin_lock_irqsave(&dsq->lock, flags);
if (dsq->nr) { scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)", dsq->id, dsq->nr); goto out_unlock_dsq; }
if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node, dsq_hash_params)) goto out_unlock_dsq;
/* * Mark dead by invalidating ->id to prevent dispatch_enqueue() from * queueing more tasks. As this function can be called from anywhere, * freeing is bounced through an irq work to avoid nesting RCU * operations inside scheduler locks. */ dsq->id = SCX_DSQ_INVALID; if (llist_add(&dsq->free_node, &dsqs_to_free)) irq_work_queue(&free_dsq_irq_work);
out_unlock_dsq: raw_spin_unlock_irqrestore(&dsq->lock, flags);out_unlock_rcu: rcu_read_unlock();}
#ifdef CONFIG_EXT_GROUP_SCHEDstatic void scx_cgroup_exit(struct scx_sched *sch){ struct cgroup_subsys_state *css;
scx_cgroup_enabled = false;
/* * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk * cgroups and exit all the inited ones, all online cgroups are exited. */ css_for_each_descendant_post(css, &root_task_group.css) { struct task_group *tg = css_tg(css);
if (!(tg->scx.flags & SCX_TG_INITED)) continue; tg->scx.flags &= ~SCX_TG_INITED;
if (!sch->ops.cgroup_exit) continue;
SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup); }}
static int scx_cgroup_init(struct scx_sched *sch){ struct cgroup_subsys_state *css; int ret;
/* * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk * cgroups and init, all online cgroups are initialized. */ css_for_each_descendant_pre(css, &root_task_group.css) { struct task_group *tg = css_tg(css); struct scx_cgroup_init_args args = { .weight = tg->scx.weight, .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, };
if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) continue;
if (!sch->ops.cgroup_init) { tg->scx.flags |= SCX_TG_INITED; continue; }
ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); if (ret) { scx_error(sch, "ops.cgroup_init() failed (%d)", ret); return ret; } tg->scx.flags |= SCX_TG_INITED; }
WARN_ON_ONCE(scx_cgroup_enabled); scx_cgroup_enabled = true;
return 0;}
#elsestatic void scx_cgroup_exit(struct scx_sched *sch) {}static int scx_cgroup_init(struct scx_sched *sch) { return 0; }#endif
/******************************************************************************** * Sysfs interface and ops enable/disable. */
#define SCX_ATTR(_name) \ static struct kobj_attribute scx_attr_##_name = { \ .attr = { .name = __stringify(_name), .mode = 0444 }, \ .show = scx_attr_##_name##_show, \ }
static ssize_t scx_attr_state_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]);}SCX_ATTR(state);
static ssize_t scx_attr_switch_all_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all));}SCX_ATTR(switch_all);
static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected));}SCX_ATTR(nr_rejected);
static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq));}SCX_ATTR(hotplug_seq);
static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq));}SCX_ATTR(enable_seq);
static struct attribute *scx_global_attrs[] = { &scx_attr_state.attr, &scx_attr_switch_all.attr, &scx_attr_nr_rejected.attr, &scx_attr_hotplug_seq.attr, &scx_attr_enable_seq.attr, NULL,};
static const struct attribute_group scx_global_attr_group = { .attrs = scx_global_attrs,};
static void free_pnode(struct scx_sched_pnode *pnode);static void free_exit_info(struct scx_exit_info *ei);
static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch){ size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); int cpu;
if (!sch->is_cid_type || !sch->arena_pool) return 0;
sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); if (!sch->set_cmask_scratch) return -ENOMEM;
for_each_possible_cpu(cpu) { struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
*slot = scx_arena_alloc(sch, size); if (!*slot) return -ENOMEM; scx_cmask_init(*slot, 0, num_possible_cpus()); } return 0;}
static void scx_set_cmask_scratch_free(struct scx_sched *sch){ size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); int cpu;
if (!sch->set_cmask_scratch) return;
for_each_possible_cpu(cpu) { struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
scx_arena_free(sch, *slot, size); } free_percpu(sch->set_cmask_scratch); sch->set_cmask_scratch = NULL;}
static void scx_sched_free_rcu_work(struct work_struct *work){ struct rcu_work *rcu_work = to_rcu_work(work); struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); struct rhashtable_iter rht_iter; struct scx_dispatch_q *dsq; int cpu, node;
irq_work_sync(&sch->disable_irq_work); kthread_destroy_worker(sch->helper); timer_shutdown_sync(&sch->bypass_lb_timer); free_cpumask_var(sch->bypass_lb_donee_cpumask); free_cpumask_var(sch->bypass_lb_resched_cpumask);
#ifdef CONFIG_EXT_SUB_SCHED kfree(sch->cgrp_path); if (sch_cgroup(sch)) cgroup_put(sch_cgroup(sch)); if (sch->sub_kset) kobject_put(&sch->sub_kset->kobj); if (scx_parent(sch)) kobject_put(&scx_parent(sch)->kobj);#endif /* CONFIG_EXT_SUB_SCHED */
for_each_possible_cpu(cpu) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
/* * $sch would have entered bypass mode before the RCU grace * period. As that blocks new deferrals, all * deferred_reenq_local_node's must be off-list by now. */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
exit_dsq(bypass_dsq(sch, cpu)); }
free_percpu(sch->pcpu);
for_each_node_state(node, N_POSSIBLE) free_pnode(sch->pnode[node]); kfree(sch->pnode);
rhashtable_walk_enter(&sch->dsq_hash, &rht_iter); do { rhashtable_walk_start(&rht_iter);
while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter)))) destroy_dsq(sch, dsq->id);
rhashtable_walk_stop(&rht_iter); } while (dsq == ERR_PTR(-EAGAIN)); rhashtable_walk_exit(&rht_iter);
rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); free_exit_info(sch->exit_info); scx_set_cmask_scratch_free(sch); scx_arena_pool_destroy(sch); if (sch->arena_map) bpf_map_put(sch->arena_map); kfree(sch);}
static void scx_kobj_release(struct kobject *kobj){ struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); queue_rcu_work(system_dfl_wq, &sch->rcu_work);}
static ssize_t scx_attr_ops_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
return sysfs_emit(buf, "%s\n", sch->ops.name);}SCX_ATTR(ops);
#define scx_attr_event_show(buf, at, events, kind) ({ \ sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind); \})
static ssize_t scx_attr_events_show(struct kobject *kobj, struct kobj_attribute *ka, char *buf){ struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); struct scx_event_stats events; int at = 0;
scx_read_events(sch, &events); at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); return at;}SCX_ATTR(events);
static struct attribute *scx_sched_attrs[] = { &scx_attr_ops.attr, &scx_attr_events.attr, NULL,};ATTRIBUTE_GROUPS(scx_sched);
static const struct kobj_type scx_ktype = { .release = scx_kobj_release, .sysfs_ops = &kobj_sysfs_ops, .default_groups = scx_sched_groups,};
static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env){ const struct scx_sched *sch;
/* * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype) * and sub-scheduler kset kobjects (kset_ktype) through the parent * chain walk. Filter out the latter to avoid invalid casts. */ if (kobj->ktype != &scx_ktype) return 0;
sch = container_of(kobj, struct scx_sched, kobj);
return add_uevent_var(env, "SCXOPS=%s", sch->ops.name);}
static const struct kset_uevent_ops scx_uevent_ops = { .uevent = scx_uevent,};
/* * Used by sched_fork() and __setscheduler_prio() to pick the matching * sched_class. dl/rt are already handled. */bool task_should_scx(int policy){ /* if disabled, nothing should be on it */ if (!scx_enabled()) return false;
/* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */ if (READ_ONCE(scx_switching_all)) return true;
/* * scx is tearing down - keep new SCHED_EXT tasks out. * * Must come after scx_switching_all test, which serves as a proxy * for __scx_switched_all. While __scx_switched_all is set, we must * return true via the branch above: a fork routed to fair would * stall because next_active_class() skips fair. * * This can develop into a deadlock - scx holds scx_enable_mutex across * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is * the stalled task, the disable path can never grab the mutex to clear * scx_switching_all. */ if (unlikely(scx_enable_state() == SCX_DISABLING)) return false;
return policy == SCHED_EXT;}
bool scx_allow_ttwu_queue(const struct task_struct *p){ struct scx_sched *sch;
if (!scx_enabled()) return true;
sch = scx_task_sched(p); if (unlikely(!sch)) return true;
if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP) return true;
if (unlikely(p->sched_class != &ext_sched_class)) return true;
return false;}
/** * handle_lockup - sched_ext common lockup handler * @fmt: format string * * Called on system stall or lockup condition and initiates abort of sched_ext * if enabled, which may resolve the reported lockup. * * Returns %true if sched_ext is enabled and abort was initiated, which may * resolve the lockup. %false if sched_ext is not enabled or abort was already * initiated by someone else. */static __printf(1, 2) bool handle_lockup(const char *fmt, ...){ struct scx_sched *sch; va_list args; bool ret;
guard(rcu)();
sch = rcu_dereference(scx_root); if (unlikely(!sch)) return false;
switch (scx_enable_state()) { case SCX_ENABLING: case SCX_ENABLED: va_start(args, fmt); ret = scx_verror(sch, fmt, args); va_end(args); return ret; default: return false; }}
/** * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler * * While there are various reasons why RCU CPU stalls can occur on a system * that may not be caused by the current BPF scheduler, try kicking out the * current scheduler in an attempt to recover the system to a good state before * issuing panics. * * Returns %true if sched_ext is enabled and abort was initiated, which may * resolve the reported RCU stall. %false if sched_ext is not enabled or someone * else already initiated abort. */bool scx_rcu_cpu_stall(void){ return handle_lockup("RCU CPU stall detected!");}
/** * scx_softlockup - sched_ext softlockup handler * @dur_s: number of seconds of CPU stuck due to soft lockup * * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can * live-lock the system by making many CPUs target the same DSQ to the point * where soft-lockup detection triggers. This function is called from * soft-lockup watchdog when the triggering point is close and tries to unjam * the system and aborting the BPF scheduler. */void scx_softlockup(u32 dur_s){ if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) return;
printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", smp_processor_id(), dur_s);}
/* * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(), * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing * it from NMI context can lead to deadlocks. Defer via irq_work; the * disable path runs off irq_work anyway. */static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1);
static void scx_hardlockup_irq_workfn(struct irq_work *work){ int cpu = atomic_xchg(&scx_hardlockup_cpu, -1);
if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", cpu);}
static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn);
/** * scx_hardlockup - sched_ext hardlockup handler * * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting * numerous affinitized tasks in a single queue and directing all CPUs at it. * Try kicking out the current scheduler in an attempt to recover the system to * a good state before taking more drastic actions. * * Queues an irq_work; the handle_lockup() call happens in IRQ context (see * scx_hardlockup_irq_workfn). * * Returns %true if sched_ext is enabled and the work was queued, %false * otherwise. */bool scx_hardlockup(int cpu){ if (!rcu_access_pointer(scx_root)) return false;
atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu); irq_work_queue(&scx_hardlockup_irq_work); return true;}
static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, struct cpumask *donee_mask, struct cpumask *resched_mask, u32 nr_donor_target, u32 nr_donee_target){ struct rq *donor_rq = cpu_rq(donor); struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); struct task_struct *p, *n; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; u32 nr_balanced = 0, min_delta_us;
/* * All we want to guarantee is reasonable forward progress. No reason to * fine tune. Assuming every task on @donor_dsq runs their full slice, * consider offloading iff the total queued duration is over the * threshold. */ min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) return 0;
raw_spin_rq_lock_irq(donor_rq); raw_spin_lock(&donor_dsq->lock); list_add(&cursor.node, &donor_dsq->list);resume: n = container_of(&cursor, struct task_struct, scx.dsq_list); n = nldsq_next_task(donor_dsq, n, false);
while ((p = n)) { struct scx_dispatch_q *donee_dsq; int donee;
n = nldsq_next_task(donor_dsq, n, false);
if (donor_dsq->nr <= nr_donor_target) break;
if (cpumask_empty(donee_mask)) break;
/* * If an earlier pass placed @p on @donor_dsq from a different * CPU and the donee hasn't consumed it yet, @p is still on the * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved * without its rq locked. Skip. */ if (task_rq(p) != donor_rq) continue;
donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); if (donee >= nr_cpu_ids) continue;
donee_dsq = bypass_dsq(sch, donee);
/* * $p's rq is not locked but $p's DSQ lock protects its * scheduling properties making this test safe. */ if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) continue;
/* * Moving $p from one non-local DSQ to another. The source rq * and DSQ are already locked. Do an abbreviated dequeue and * then perform enqueue without unlocking $donor_dsq. * * We don't want to drop and reacquire the lock on each * iteration as @donor_dsq can be very long and potentially * highly contended. Donee DSQs are less likely to be contended. * The nested locking is safe as only this LB moves tasks * between bypass DSQs. */ dispatch_dequeue_locked(p, donor_dsq); dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED);
/* * $donee might have been idle and need to be woken up. No need * to be clever. Kick every CPU that receives tasks. */ cpumask_set_cpu(donee, resched_mask);
if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) cpumask_clear_cpu(donee, donee_mask);
nr_balanced++; if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { list_move_tail(&cursor.node, &n->scx.dsq_list.node); raw_spin_unlock(&donor_dsq->lock); raw_spin_rq_unlock_irq(donor_rq); cpu_relax(); raw_spin_rq_lock_irq(donor_rq); raw_spin_lock(&donor_dsq->lock); goto resume; } }
list_del_init(&cursor.node); raw_spin_unlock(&donor_dsq->lock); raw_spin_rq_unlock_irq(donor_rq);
return nr_balanced;}
static void bypass_lb_node(struct scx_sched *sch, int node){ const struct cpumask *node_mask = cpumask_of_node(node); struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; u32 nr_target, nr_donor_target; u32 before_min = U32_MAX, before_max = 0; u32 after_min = U32_MAX, after_max = 0; int cpu;
/* count the target tasks and CPUs */ for_each_cpu_and(cpu, cpu_online_mask, node_mask) { u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr);
nr_tasks += nr; nr_cpus++;
before_min = min(nr, before_min); before_max = max(nr, before_max); }
if (!nr_cpus) return;
/* * We don't want CPUs to have more than $nr_donor_target tasks and * balancing to fill donee CPUs upto $nr_target. Once targets are * calculated, find the donee CPUs. */ nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100);
cpumask_clear(donee_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) cpumask_set_cpu(cpu, donee_mask); }
/* iterate !donee CPUs and see if they should be offloaded */ cpumask_clear(resched_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { if (cpumask_empty(donee_mask)) break; if (cpumask_test_cpu(cpu, donee_mask)) continue; if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) continue;
nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, nr_donor_target, nr_target); }
for_each_cpu(cpu, resched_mask) resched_cpu(cpu);
for_each_cpu_and(cpu, cpu_online_mask, node_mask) { u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr);
after_min = min(nr, after_min); after_max = max(nr, after_max);
}
trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced, before_min, before_max, after_min, after_max);}
/* * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some * bypass DSQs can be overloaded. If there are enough tasks to saturate other * lightly loaded CPUs, such imbalance can lead to very high execution latency * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such * outcomes, a simple load balancing mechanism is implemented by the following * timer which runs periodically while bypass mode is in effect. */static void scx_bypass_lb_timerfn(struct timer_list *timer){ struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer); int node; u32 intv_us;
if (!bypass_dsp_enabled(sch)) return;
for_each_node_with_cpus(node) bypass_lb_node(sch, node);
intv_us = READ_ONCE(scx_bypass_lb_intv_us); if (intv_us) mod_timer(timer, jiffies + usecs_to_jiffies(intv_us));}
static bool inc_bypass_depth(struct scx_sched *sch){ lockdep_assert_held(&scx_bypass_lock);
WARN_ON_ONCE(sch->bypass_depth < 0); WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1); if (sch->bypass_depth != 1) return false;
WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); sch->bypass_timestamp = ktime_get_ns(); scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); return true;}
static bool dec_bypass_depth(struct scx_sched *sch){ lockdep_assert_held(&scx_bypass_lock);
WARN_ON_ONCE(sch->bypass_depth < 1); WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1); if (sch->bypass_depth != 0) return false;
WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL); scx_add_event(sch, SCX_EV_BYPASS_DURATION, ktime_get_ns() - sch->bypass_timestamp); return true;}
static void enable_bypass_dsp(struct scx_sched *sch){ struct scx_sched *host = scx_parent(sch) ?: sch; u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); s32 ret;
/* * @sch->bypass_depth transitioning from 0 to 1 triggers enabling. * Shouldn't stagger. */ if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim))) return;
/* * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of * the nearest non-bypassing ancestor or root. As enable_bypass_dsp() is * called iff @sch is not already bypassed due to an ancestor bypassing, * we can assume that the parent is not bypassing and thus will be the * host of the bypass DSQs. * * While the situation may change in the future, the following * guarantees that the nearest non-bypassing ancestor or root has bypass * dispatch enabled while a descendant is bypassing, which is all that's * required. * * bypass_dsp_enabled() test is used to determine whether to enter the * bypass dispatch handling path from both bypassing and hosting scheds. * Bump enable depth on both @sch and bypass dispatch host. */ ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); WARN_ON_ONCE(ret <= 0);
if (host != sch) { ret = atomic_inc_return(&host->bypass_dsp_enable_depth); WARN_ON_ONCE(ret <= 0); }
/* * The LB timer will stop running if bypass dispatch is disabled. Start * after enabling bypass dispatch. */ if (intv_us && !timer_pending(&host->bypass_lb_timer)) mod_timer(&host->bypass_lb_timer, jiffies + usecs_to_jiffies(intv_us));}
/* may be called without holding scx_bypass_lock */static void disable_bypass_dsp(struct scx_sched *sch){ s32 ret;
if (!test_and_clear_bit(0, &sch->bypass_dsp_claim)) return;
ret = atomic_dec_return(&sch->bypass_dsp_enable_depth); WARN_ON_ONCE(ret < 0);
if (scx_parent(sch)) { ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth); WARN_ON_ONCE(ret < 0); }}
/** * scx_bypass - [Un]bypass scx_ops and guarantee forward progress * @sch: sched to bypass * @bypass: true for bypass, false for unbypass * * Bypassing guarantees that all runnable tasks make forward progress without * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might * be held by tasks that the BPF scheduler is forgetting to run, which * unfortunately also excludes toggling the static branches. * * Let's work around by overriding a couple ops and modifying behaviors based on * the DISABLING state and then cycling the queued tasks through dequeue/enqueue * to force global FIFO scheduling. * * - ops.select_cpu() is ignored and the default select_cpu() is used. * * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. * %SCX_OPS_ENQ_LAST is also ignored. * * - ops.dispatch() is ignored. * * - balance_one() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue with core_sched_at touched. * * - pick_next_task() suppresses zero slice warning. * * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM * operations. * * - scx_prio_less() reverts to the default core_sched_at order. */static void scx_bypass(struct scx_sched *sch, bool bypass){ struct scx_sched *pos; unsigned long flags; int cpu;
raw_spin_lock_irqsave(&scx_bypass_lock, flags);
if (bypass) { if (!inc_bypass_depth(sch)) goto unlock;
enable_bypass_dsp(sch); } else { if (!dec_bypass_depth(sch)) goto unlock; }
/* * Bypass state is propagated to all descendants - an scx_sched bypasses * if itself or any of its ancestors are in bypass mode. */ raw_spin_lock(&scx_sched_lock); scx_for_each_descendant_pre(pos, sch) { if (pos == sch) continue; if (bypass) inc_bypass_depth(pos); else dec_bypass_depth(pos); } raw_spin_unlock(&scx_sched_lock);
/* * No task property is changing. We just need to make sure all currently * queued tasks are re-queued according to the new scx_bypassing() * state. As an optimization, walk each rq's runnable_list instead of * the scx_tasks list. * * This function can't trust the scheduler and thus can't use * cpus_read_lock(). Walk all possible CPUs instead of online. */ for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); struct task_struct *p, *n;
raw_spin_rq_lock(rq); raw_spin_lock(&scx_sched_lock);
scx_for_each_descendant_pre(pos, sch) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
if (pos->bypass_depth) pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; else pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; }
raw_spin_unlock(&scx_sched_lock);
/* * We need to guarantee that no tasks are on the BPF scheduler * while bypassing. Either we see enabled or the enable path * sees scx_bypassing() before moving tasks to SCX. */ if (!scx_enabled()) { raw_spin_rq_unlock(rq); continue; }
/* * The use of list_for_each_entry_safe_reverse() is required * because each task is going to be removed from and added back * to the runnable_list during iteration. Because they're added * to the tail of the list, safe reverse iteration can still * visit all nodes. */ list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { if (!scx_is_descendant(scx_task_sched(p), sch)) continue;
/* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; } }
/* resched to restore ticks and idle state */ if (cpu_online(cpu) || cpu == smp_processor_id()) resched_curr(rq);
raw_spin_rq_unlock(rq); }
/* disarming must come after moving all tasks out of the bypass DSQs */ if (!bypass) disable_bypass_dsp(sch);unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);}
static void free_exit_info(struct scx_exit_info *ei){ kvfree(ei->dump); kfree(ei->msg); kfree(ei->bt); kfree(ei);}
static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len){ struct scx_exit_info *ei;
ei = kzalloc_obj(*ei); if (!ei) return NULL;
ei->exit_cpu = -1; ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN); ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL);
if (!ei->bt || !ei->msg || !ei->dump) { free_exit_info(ei); return NULL; }
return ei;}
static const char *scx_exit_reason(enum scx_exit_kind kind){ switch (kind) { case SCX_EXIT_UNREG: return "unregistered from user space"; case SCX_EXIT_UNREG_BPF: return "unregistered from BPF"; case SCX_EXIT_UNREG_KERN: return "unregistered from the main kernel"; case SCX_EXIT_SYSRQ: return "disabled by sysrq-S"; case SCX_EXIT_PARENT: return "parent exiting"; case SCX_EXIT_ERROR: return "runtime error"; case SCX_EXIT_ERROR_BPF: return "scx_bpf_error"; case SCX_EXIT_ERROR_STALL: return "runnable task stall"; default: return "<UNKNOWN>"; }}
static void free_kick_syncs(void){ int cpu;
for_each_possible_cpu(cpu) { struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); struct scx_kick_syncs *to_free;
to_free = rcu_replace_pointer(*ksyncs, NULL, true); if (to_free) kvfree_rcu(to_free, rcu); }}
static void refresh_watchdog(void){ struct scx_sched *sch; unsigned long intv = ULONG_MAX;
/* take the shortest timeout and use its half for watchdog interval */ rcu_read_lock(); list_for_each_entry_rcu(sch, &scx_sched_all, all) intv = max(min(intv, sch->watchdog_timeout / 2), 1); rcu_read_unlock();
WRITE_ONCE(scx_watchdog_timestamp, jiffies); WRITE_ONCE(scx_watchdog_interval, intv);
if (intv < ULONG_MAX) mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv); else cancel_delayed_work_sync(&scx_watchdog_work);}
static s32 scx_link_sched(struct scx_sched *sch){ const char *err_msg = ""; s32 ret = 0;
scoped_guard(raw_spinlock_irq, &scx_sched_lock) {#ifdef CONFIG_EXT_SUB_SCHED struct scx_sched *parent = scx_parent(sch);
if (parent) { /* * scx_claim_exit() propagates exit_kind transition to * its sub-scheds while holding scx_sched_lock - either * we can see the parent's non-NONE exit_kind or the * parent can shoot us down. */ if (atomic_read(&parent->exit_kind) != SCX_EXIT_NONE) { err_msg = "parent disabled"; ret = -ENOENT; break; }
ret = rhashtable_lookup_insert_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); if (ret) { err_msg = "failed to insert into scx_sched_hash"; break; }
list_add_tail(&sch->sibling, &parent->children); }#endif /* CONFIG_EXT_SUB_SCHED */
list_add_tail_rcu(&sch->all, &scx_sched_all); }
/* * scx_error() takes scx_sched_lock via scx_claim_exit(), so it must run after * the guard above is released. */ if (ret) { scx_error(sch, "%s (%d)", err_msg, ret); return ret; }
refresh_watchdog(); return 0;}
static void scx_unlink_sched(struct scx_sched *sch){ scoped_guard(raw_spinlock_irq, &scx_sched_lock) {#ifdef CONFIG_EXT_SUB_SCHED if (scx_parent(sch)) { rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node, scx_sched_hash_params); list_del_init(&sch->sibling); }#endif /* CONFIG_EXT_SUB_SCHED */ list_del_rcu(&sch->all); }
refresh_watchdog();}
/* * Called to disable future dumps and wait for in-progress one while disabling * @sch. Once @sch becomes empty during disable, there's no point in dumping it. * This prevents calling dump ops on a dead sch. */static void scx_disable_dump(struct scx_sched *sch){ guard(raw_spinlock_irqsave)(&scx_dump_lock); sch->dump_disabled = true;}
static void scx_log_sched_disable(struct scx_sched *sch){ struct scx_exit_info *ei = sch->exit_info; const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler";
if (ei->kind >= SCX_EXIT_ERROR) { pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, sch->ops.name, ei->reason);
if (ei->msg[0] != '\0') pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg);#ifdef CONFIG_STACKTRACE stack_trace_print(ei->bt, ei->bt_len, 2);#endif } else { pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type, sch->ops.name, ei->reason); }}
#ifdef CONFIG_EXT_SUB_SCHEDstatic DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
static void drain_descendants(struct scx_sched *sch){ /* * Child scheds that finished the critical part of disabling will take * themselves off @sch->children. Wait for it to drain. As propagation * is recursive, empty @sch->children means that all proper descendant * scheds reached unlinking stage. */ wait_event(scx_unlink_waitq, list_empty(&sch->children));}
static void scx_fail_parent(struct scx_sched *sch, struct task_struct *failed, s32 fail_code){ struct scx_sched *parent = scx_parent(sch); struct scx_task_iter sti; struct task_struct *p;
scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", fail_code, failed->comm, failed->pid);
/* * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into * it. This may cause downstream failures on the BPF side but $parent is * dying anyway. */ scx_bypass(parent, true);
scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { if (scx_task_on_sched(parent, p)) continue;
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { scx_disable_and_exit_task(sch, p); scx_set_task_sched(p, parent); } } scx_task_iter_stop(&sti);}
static void scx_sub_disable(struct scx_sched *sch){ struct scx_sched *parent = scx_parent(sch); struct scx_task_iter sti; struct task_struct *p; int ret;
/* * Guarantee forward progress and wait for descendants to be disabled. * To limit disruptions, $parent is not bypassed. Tasks are fully * prepped and then inserted back into $parent. */ scx_bypass(sch, true); drain_descendants(sch);
/* * Here, every runnable task is guaranteed to make forward progress and * we can safely use blocking synchronization constructs. Actually * disable ops. */ mutex_lock(&scx_enable_mutex); percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock();
set_cgroup_sched(sch_cgroup(sch), parent);
scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { struct rq *rq; struct rq_flags rf;
/* filter out duplicate visits */ if (scx_task_on_sched(parent, p)) continue;
/* * By the time control reaches here, all descendant schedulers * should already have been disabled. */ WARN_ON_ONCE(!scx_task_on_sched(sch, p));
/* * @p is pinned by the iter: css_task_iter_next() takes a * reference and holds it until the next iter_next() call, so * @p->usage is guaranteed > 0. */ get_task_struct(p);
scx_task_iter_unlock(&sti);
/* * $p is READY or ENABLED on @sch. Initialize for $parent, * disable and exit from @sch, and then switch over to $parent. * * If a task fails to initialize for $parent, the only available * action is disabling $parent too. While this allows disabling * of a child sched to cause the parent scheduler to fail, the * failure can only originate from ops.init_task() of the * parent. A child can't directly affect the parent through its * own failures. */ ret = __scx_init_task(parent, p, false); if (ret) { scx_fail_parent(sch, p, ret); put_task_struct(p); break; }
rq = task_rq_lock(p, &rf);
if (scx_get_task_state(p) == SCX_TASK_DEAD) { /* * sched_ext_dead() raced us between __scx_init_task() * and this rq lock and ran exit_task() on @sch (the * sched @p was on at that point), not on $parent. * $parent's just-completed init is owed an exit_task() * and we issue it here. */ scx_sub_init_cancel_task(parent, p); task_rq_unlock(rq, p, &rf); put_task_struct(p); continue; }
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* * $p is initialized for $parent and still attached to * @sch. Disable and exit for @sch, switch over to * $parent, override the state to READY to account for * $p having already been initialized, and then enable. */ scx_disable_and_exit_task(sch, p); scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_set_task_state(p, SCX_TASK_INIT); scx_set_task_sched(p, parent); scx_set_task_state(p, SCX_TASK_READY); scx_enable_task(parent, p); }
task_rq_unlock(rq, p, &rf); put_task_struct(p); } scx_task_iter_stop(&sti);
scx_disable_dump(sch);
scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem);
/* * All tasks are moved off of @sch but there may still be on-going * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use * the expedited version as ancestors may be waiting in bypass mode. * Also, tell the parent that there is no need to keep running bypass * DSQs for us. */ synchronize_rcu_expedited(); disable_bypass_dsp(sch);
scx_unlink_sched(sch);
mutex_unlock(&scx_enable_mutex);
/* * @sch is now unlinked from the parent's children list. Notify and call * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called * after unlinking and releasing all locks. See scx_claim_exit(). */ wake_up_all(&scx_unlink_waitq);
if (parent->ops.sub_detach && sch->sub_attached) { struct scx_sub_detach_args sub_detach_args = { .ops = &sch->ops, .cgroup_path = sch->cgrp_path, }; SCX_CALL_OP(parent, sub_detach, NULL, &sub_detach_args); }
scx_log_sched_disable(sch);
if (sch->ops.exit) SCX_CALL_OP(sch, exit, NULL, sch->exit_info); if (sch->sub_kset) kobject_del(&sch->sub_kset->kobj); kobject_del(&sch->kobj);}#else /* CONFIG_EXT_SUB_SCHED */static inline void drain_descendants(struct scx_sched *sch) { }static inline void scx_sub_disable(struct scx_sched *sch) { }#endif /* CONFIG_EXT_SUB_SCHED */
static void scx_root_disable(struct scx_sched *sch){ struct scx_task_iter sti; struct task_struct *p; bool was_switched_all; int cpu;
/* guarantee forward progress and wait for descendants to be disabled */ scx_bypass(sch, true); drain_descendants(sch);
switch (scx_set_enable_state(SCX_DISABLING)) { case SCX_DISABLING: WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); break; case SCX_DISABLED: pr_warn("sched_ext: ops error detected without ops (%s)\n", sch->exit_info->msg); WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); goto done; default: break; }
/* * Here, every runnable task is guaranteed to make forward progress and * we can safely use blocking synchronization constructs. Actually * disable ops. */ mutex_lock(&scx_enable_mutex);
was_switched_all = scx_switched_all();
static_branch_disable(&__scx_switched_all); WRITE_ONCE(scx_switching_all, false);
/* * Shut down cgroup support before tasks so that the cgroup attach path * doesn't race against scx_disable_and_exit_task(). */ scx_cgroup_lock(); scx_cgroup_exit(sch); scx_cgroup_unlock();
/* * The BPF scheduler is going away. All tasks including %TASK_DEAD ones * must be switched out and exited synchronously. */ percpu_down_write(&scx_fork_rwsem);
scx_init_task_enabled = false;
scx_task_iter_start(&sti, NULL); while ((p = scx_task_iter_next_locked(&sti))) { unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; const struct sched_class *old_class = p->sched_class; const struct sched_class *new_class = scx_setscheduler_class(p);
update_rq_clock(task_rq(p));
if (old_class != new_class) queue_flags |= DEQUEUE_CLASS;
scoped_guard (sched_change, p, queue_flags) { p->sched_class = new_class; }
scx_disable_and_exit_task(scx_task_sched(p), p); } scx_task_iter_stop(&sti);
scx_disable_dump(sch);
scx_cgroup_lock(); set_cgroup_sched(sch_cgroup(sch), NULL); scx_cgroup_unlock();
percpu_up_write(&scx_fork_rwsem);
/* * Invalidate all the rq clocks to prevent getting outdated * rq clocks from a previous scx scheduler. * * Also re-balance the dl_server bandwidth reservations: detach * ext_server (no more sched_ext tasks) and reinstate fair_server if it * was previously detached because we were running in full mode. * * Unlike the enable path, this runs on a recovery path that cannot * fail, so we use dl_server_swap_bw() to atomically free ext_server's * bandwidth and reclaim it for fair_server under the same dl_b lock. * * The swap can still fail with -EBUSY if someone bumped ext_server's * runtime via debugfs between enable and disable; in that narrow case * both servers end up detached and we just WARN. */ for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu);
scx_rq_clock_invalidate(rq);
scoped_guard(rq_lock_irqsave, rq) { update_rq_clock(rq); if (was_switched_all) { if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server, &rq->fair_server))) pr_warn("failed to re-attach fair_server on CPU %d\n", cpu); } else { dl_server_detach_bw(&rq->ext_server); } } }
/* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); static_branch_disable(&__scx_is_cid_type); if (sch->ops.flags & SCX_OPS_TID_TO_TASK) static_branch_disable(&__scx_tid_to_task_enabled); bitmap_zero(sch->has_op, SCX_OPI_END); scx_idle_disable(); synchronize_rcu(); if (sch->ops.flags & SCX_OPS_TID_TO_TASK) rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
scx_log_sched_disable(sch);
if (sch->ops.exit) SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
scx_unlink_sched(sch);
/* * scx_root clearing must be inside cpus_read_lock(). See * handle_hotplug(). */ cpus_read_lock(); RCU_INIT_POINTER(scx_root, NULL); cpus_read_unlock();
/* * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs * could observe an object of the same name still in the hierarchy when * the next scheduler is loaded. */#ifdef CONFIG_EXT_SUB_SCHED if (sch->sub_kset) kobject_del(&sch->sub_kset->kobj);#endif kobject_del(&sch->kobj);
free_kick_syncs();
mutex_unlock(&scx_enable_mutex);
WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);done: scx_bypass(sch, false);}
/* * Claim the exit on @sch. The caller must ensure that the helper kthread work * is kicked before the current task can be preempted. Once exit_kind is * claimed, scx_error() can no longer trigger, so if the current task gets * preempted and the BPF scheduler fails to schedule it back, the helper work * will never be kicked and the whole system can wedge. */static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind){ int none = SCX_EXIT_NONE;
lockdep_assert_preemption_disabled();
if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) kind = SCX_EXIT_ERROR;
if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) return false;
/* * Some CPUs may be trapped in the dispatch paths. Set the aborting * flag to break potential live-lock scenarios, ensuring we can * successfully reach scx_bypass(). */ WRITE_ONCE(sch->aborting, true);
/* * Propagate exits to descendants immediately. Each has a dedicated * helper kthread and can run in parallel. While most of disabling is * serialized, running them in separate threads allows parallelizing * ops.exit(), which can take arbitrarily long prolonging bypass mode. * * To guarantee forward progress, this propagation must be in-line so * that ->aborting is synchronously asserted for all sub-scheds. The * propagation is also the interlocking point against sub-sched * attachment. See scx_link_sched(). * * This doesn't cause recursions as propagation only takes place for * non-propagation exits. */ if (kind != SCX_EXIT_PARENT) { scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) { struct scx_sched *pos; scx_for_each_descendant_pre(pos, sch) scx_disable(pos, SCX_EXIT_PARENT); } }
return true;}
static void scx_disable_workfn(struct kthread_work *work){ struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); struct scx_exit_info *ei = sch->exit_info; int kind;
kind = atomic_read(&sch->exit_kind); while (true) { if (kind == SCX_EXIT_DONE) /* already disabled? */ return; WARN_ON_ONCE(kind == SCX_EXIT_NONE); if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) break; } ei->kind = kind; ei->reason = scx_exit_reason(ei->kind);
if (scx_parent(sch)) scx_sub_disable(sch); else scx_root_disable(sch);}
static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind){ guard(preempt)(); if (scx_claim_exit(sch, kind)) irq_work_queue(&sch->disable_irq_work);}
/** * scx_flush_disable_work - flush the disable work and wait for it to finish * @sch: the scheduler * * sch->disable_work might still not queued, causing kthread_flush_work() * as a noop. Syncing the irq_work first is required to guarantee the * kthread work has been queued before waiting for it. */static void scx_flush_disable_work(struct scx_sched *sch){ int kind;
do { irq_work_sync(&sch->disable_irq_work); kthread_flush_work(&sch->disable_work); kind = atomic_read(&sch->exit_kind); } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);}
static void dump_newline(struct seq_buf *s){ trace_sched_ext_dump("");
/* @s may be zero sized and seq_buf triggers WARN if so */ if (s->size) seq_buf_putc(s, '\n');}
static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...){ va_list args;
#ifdef CONFIG_TRACEPOINTS if (trace_sched_ext_dump_enabled()) { /* protected by scx_dump_lock */ static char line_buf[SCX_EXIT_MSG_LEN];
va_start(args, fmt); vscnprintf(line_buf, sizeof(line_buf), fmt, args); va_end(args);
trace_call__sched_ext_dump(line_buf); }#endif /* @s may be zero sized and seq_buf triggers WARN if so */ if (s->size) { va_start(args, fmt); seq_buf_vprintf(s, fmt, args); va_end(args);
seq_buf_putc(s, '\n'); }}
static void dump_stack_trace(struct seq_buf *s, const char *prefix, const unsigned long *bt, unsigned int len){ unsigned int i;
for (i = 0; i < len; i++) dump_line(s, "%s%pS", prefix, (void *)bt[i]);}
static void ops_dump_init(struct seq_buf *s, const char *prefix){ struct scx_dump_data *dd = &scx_dump_data;
lockdep_assert_irqs_disabled();
dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ dd->first = true; dd->cursor = 0; dd->s = s; dd->prefix = prefix;}
static void ops_dump_flush(void){ struct scx_dump_data *dd = &scx_dump_data; char *line = dd->buf.line;
if (!dd->cursor) return;
/* * There's something to flush and this is the first line. Insert a blank * line to distinguish ops dump. */ if (dd->first) { dump_newline(dd->s); dd->first = false; }
/* * There may be multiple lines in $line. Scan and emit each line * separately. */ while (true) { char *end = line; char c;
while (*end != '\n' && *end != '\0') end++;
/* * If $line overflowed, it may not have newline at the end. * Always emit with a newline. */ c = *end; *end = '\0'; dump_line(dd->s, "%s%s", dd->prefix, line); if (c == '\0') break;
/* move to the next line */ end++; if (*end == '\0') break; line = end; }
dd->cursor = 0;}
static void ops_dump_exit(void){ ops_dump_flush(); scx_dump_data.cpu = -1;}
static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, struct rq *rq, struct task_struct *p, char marker){ static unsigned long bt[SCX_EXIT_BT_LEN]; struct scx_sched *task_sch = scx_task_sched(p); const char *own_marker; char sch_id_buf[32]; char dsq_id_buf[19] = "(n/a)"; unsigned long ops_state = atomic_long_read(&p->scx.ops_state); unsigned int bt_len = 0;
own_marker = task_sch == sch ? "*" : "";
if (task_sch->level == 0) scnprintf(sch_id_buf, sizeof(sch_id_buf), "root"); else scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu", task_sch->level, task_sch->ops.sub_cgroup_id);
if (p->scx.dsq) scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", (unsigned long long)p->scx.dsq->id);
dump_newline(s); dump_line(s, " %c%c %s[%d] %s%s %+ldms", marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf, jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT, p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT); dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s", p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf); dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u", p->scx.dsq_vtime, p->scx.slice, p->scx.weight); dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr), p->migration_disabled);
if (SCX_HAS_OP(sch, dump_task)) { ops_dump_init(s, " "); SCX_CALL_OP(sch, dump_task, rq, dctx, p); ops_dump_exit(); }
#ifdef CONFIG_STACKTRACE bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1);#endif if (bt_len) { dump_newline(s); dump_stack_trace(s, " ", bt, bt_len); }}
static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx, int cpu, bool dump_all_tasks){ struct rq *rq = cpu_rq(cpu); struct rq_flags rf; struct task_struct *p; struct seq_buf ns; size_t avail, used; char *buf; bool idle;
rq_lock_irqsave(rq, &rf);
idle = list_empty(&rq->scx.runnable_list) && rq->curr->sched_class == &idle_sched_class;
if (idle && !SCX_HAS_OP(sch, dump_cpu)) goto next;
/* * We don't yet know whether ops.dump_cpu() will produce output * and we may want to skip the default CPU dump if it doesn't. * Use a nested seq_buf to generate the standard dump so that we * can decide whether to commit later. */ avail = seq_buf_get_buf(s, &buf); seq_buf_init(&ns, buf, avail);
dump_newline(&ns); dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released, rq->scx.ops_qseq, rq->scx.kick_sync); dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); if (!cpumask_empty(rq->scx.cpus_to_kick)) dump_line(&ns, " cpus_to_kick : %*pb", cpumask_pr_args(rq->scx.cpus_to_kick)); if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) dump_line(&ns, " idle_to_kick : %*pb", cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); if (!cpumask_empty(rq->scx.cpus_to_preempt)) dump_line(&ns, " cpus_to_preempt: %*pb", cpumask_pr_args(rq->scx.cpus_to_preempt)); if (!cpumask_empty(rq->scx.cpus_to_wait)) dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(rq->scx.cpus_to_wait)); if (!cpumask_empty(rq->scx.cpus_to_sync)) dump_line(&ns, " cpus_to_sync : %*pb", cpumask_pr_args(rq->scx.cpus_to_sync));
used = seq_buf_used(&ns); if (SCX_HAS_OP(sch, dump_cpu)) { ops_dump_init(&ns, " "); SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle); ops_dump_exit(); }
/* * If idle && nothing generated by ops.dump_cpu(), there's * nothing interesting. Skip. */ if (idle && used == seq_buf_used(&ns)) goto next;
/* * $s may already have overflowed when $ns was created. If so, * calling commit on it will trigger BUG. */ if (avail) { seq_buf_commit(s, seq_buf_used(&ns)); if (seq_buf_has_overflowed(&ns)) seq_buf_set_overflow(s); }
if (rq->curr->sched_class == &ext_sched_class && (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) if (dump_all_tasks || scx_task_on_sched(sch, p)) scx_dump_task(sch, s, dctx, rq, p, ' ');next: rq_unlock_irqrestore(rq, &rf);}
/* * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless * of which scheduler they belong to. If false, only dump tasks owned by @sch. * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped * separately. For error dumps, @dump_all_tasks=true since only the failing * scheduler is dumped. */static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, size_t dump_len, bool dump_all_tasks){ static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; struct scx_dump_ctx dctx = { .kind = ei->kind, .exit_code = ei->exit_code, .reason = ei->reason, .at_ns = ktime_get_ns(), .at_jiffies = jiffies, }; struct seq_buf s; struct scx_event_stats events; int cpu;
guard(raw_spinlock_irqsave)(&scx_dump_lock);
if (sch->dump_disabled) return;
seq_buf_init(&s, ei->dump, dump_len);
#ifdef CONFIG_EXT_SUB_SCHED if (sch->level == 0) dump_line(&s, "%s: root", sch->ops.name); else dump_line(&s, "%s: sub%d-%llu %s", sch->ops.name, sch->level, sch->ops.sub_cgroup_id, sch->cgrp_path);#endif if (ei->kind == SCX_EXIT_NONE) { dump_line(&s, "Debug dump triggered by %s", ei->reason); } else { if (ei->exit_cpu >= 0) dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:", current->comm, current->pid, ei->kind, ei->exit_cpu); else dump_line(&s, "%s[%d] triggered exit kind %d:", current->comm, current->pid, ei->kind); dump_line(&s, " %s (%s)", ei->reason, ei->msg); dump_newline(&s); dump_line(&s, "Backtrace:"); dump_stack_trace(&s, " ", ei->bt, ei->bt_len); }
if (SCX_HAS_OP(sch, dump)) { ops_dump_init(&s, ""); SCX_CALL_OP(sch, dump, NULL, &dctx); ops_dump_exit(); }
dump_newline(&s); dump_line(&s, "CPU states"); dump_line(&s, "----------");
/* * Dump the exit CPU first so it isn't lost to dump truncation, then * walk the rest in order, skipping the one already dumped. */ if (ei->exit_cpu >= 0) scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); for_each_possible_cpu(cpu) { if (cpu != ei->exit_cpu) scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); }
dump_newline(&s); dump_line(&s, "Event counters"); dump_line(&s, "--------------");
scx_read_events(sch, &events); scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH);
if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) memcpy(ei->dump + dump_len - sizeof(trunc_marker), trunc_marker, sizeof(trunc_marker));}
static void scx_disable_irq_workfn(struct irq_work *irq_work){ struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work); struct scx_exit_info *ei = sch->exit_info;
if (ei->kind >= SCX_EXIT_ERROR) scx_dump_state(sch, ei, sch->ops.exit_dump_len, true);
kthread_queue_work(sch->helper, &sch->disable_work);}
bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, va_list args){ struct scx_exit_info *ei = sch->exit_info;
guard(preempt)();
if (!scx_claim_exit(sch, kind)) return false;
ei->exit_code = exit_code;#ifdef CONFIG_STACKTRACE if (kind >= SCX_EXIT_ERROR) ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);#endif vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
/* * Set ei->kind and ->reason for scx_dump_state(). They'll be set again * in scx_disable_workfn(). */ ei->kind = kind; ei->reason = scx_exit_reason(ei->kind); ei->exit_cpu = exit_cpu;
irq_work_queue(&sch->disable_irq_work); return true;}
static int alloc_kick_syncs(void){ int cpu;
/* * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size * can exceed percpu allocator limits on large machines. */ for_each_possible_cpu(cpu) { struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); struct scx_kick_syncs *new_ksyncs;
WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids), GFP_KERNEL, cpu_to_node(cpu)); if (!new_ksyncs) { free_kick_syncs(); return -ENOMEM; }
rcu_assign_pointer(*ksyncs, new_ksyncs); }
return 0;}
static void free_pnode(struct scx_sched_pnode *pnode){ if (!pnode) return; exit_dsq(&pnode->global_dsq); kfree(pnode);}
static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node){ struct scx_sched_pnode *pnode;
pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node); if (!pnode) return NULL;
if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) { kfree(pnode); return NULL; }
return pnode;}
/* * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid * starvation. During the READY -> ENABLED task switching loop, the calling * thread's sched_class gets switched from fair to ext. As fair has higher * priority than ext, the calling thread can be indefinitely starved under * fair-class saturation, leading to a system hang. */struct scx_enable_cmd { struct kthread_work work; union { struct sched_ext_ops *ops; struct sched_ext_ops_cid *ops_cid; }; bool is_cid_type; struct bpf_map *arena_map; /* arena ref to transfer to sch */ int ret;};
/* * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. */static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, struct cgroup *cgrp, struct scx_sched *parent){ struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; s32 level = parent ? parent->level + 1 : 0; s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids;
sch = kzalloc_flex(*sch, ancestors, level + 1); if (!sch) { ret = -ENOMEM; goto err_put_cgrp; }
sch->exit_info = alloc_exit_info(ops->exit_dump_len); if (!sch->exit_info) { ret = -ENOMEM; goto err_free_sch; }
ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params); if (ret < 0) goto err_free_ei;
sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids); if (!sch->pnode) { ret = -ENOMEM; goto err_free_hash; }
for_each_node_state(node, N_POSSIBLE) { sch->pnode[node] = alloc_pnode(sch, node); if (!sch->pnode[node]) { ret = -ENOMEM; goto err_free_pnode; } }
sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu, dsp_ctx.buf, sch->dsp_max_batch), __alignof__(struct scx_sched_pcpu)); if (!sch->pcpu) { ret = -ENOMEM; goto err_free_pnode; }
for_each_possible_cpu(cpu) { ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); if (ret) { bypass_fail_cpu = cpu; goto err_free_pcpu; } }
for_each_possible_cpu(cpu) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
pcpu->sch = sch; INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node); }
sch->helper = kthread_run_worker(0, "sched_ext_helper"); if (IS_ERR(sch->helper)) { ret = PTR_ERR(sch->helper); goto err_free_pcpu; }
sched_set_fifo(sch->helper->task);
if (parent) memcpy(sch->ancestors, parent->ancestors, level * sizeof(parent->ancestors[0])); sch->ancestors[level] = sch; sch->level = level;
if (ops->timeout_ms) sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms); else sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT;
sch->slice_dfl = SCX_SLICE_DFL; atomic_set(&sch->exit_kind, SCX_EXIT_NONE); sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn); kthread_init_work(&sch->disable_work, scx_disable_workfn); timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) { ret = -ENOMEM; goto err_stop_helper; } if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) { ret = -ENOMEM; goto err_free_lb_cpumask; } /* * Copy ops through the right union view. For cid-form the source is * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ * cpu_release; those stay zero from kzalloc. */ if (cmd->is_cid_type) { sch->ops_cid = *cmd->ops_cid; sch->is_cid_type = true; } else { sch->ops = *cmd->ops; }
#ifdef CONFIG_EXT_SUB_SCHED char *buf = kzalloc(PATH_MAX, GFP_KERNEL); if (!buf) { ret = -ENOMEM; goto err_free_lb_resched; } cgroup_path(cgrp, buf, PATH_MAX); sch->cgrp_path = kstrdup(buf, GFP_KERNEL); kfree(buf); if (!sch->cgrp_path) { ret = -ENOMEM; goto err_free_lb_resched; }
sch->cgrp = cgrp; INIT_LIST_HEAD(&sch->children); INIT_LIST_HEAD(&sch->sibling);#endif /* CONFIG_EXT_SUB_SCHED */
/* * Publishing makes @sch visible to scx_prog_sched() readers. Failure * paths after this point must free @sch through kobject_put() whose * release path defers the actual freeing by an RCU grace period. */ rcu_assign_pointer(ops->priv, sch);
sch->kobj.kset = scx_kset; INIT_LIST_HEAD(&sch->all);
#ifdef CONFIG_EXT_SUB_SCHED if (parent) { /* * Pin @parent for @sch's lifetime. The kobject hierarchy pins * it only via @parent->sub_kset, which is dropped during * disable. Released in scx_sched_free_rcu_work(). */ kobject_get(&parent->kobj); ret = kobject_init_and_add(&sch->kobj, &scx_ktype, &parent->sub_kset->kobj, "sub-%llu", cgroup_id(cgrp)); } else { ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); }
if (ret < 0) { RCU_INIT_POINTER(ops->priv, NULL); kobject_put(&sch->kobj); return ERR_PTR(ret); }
if (ops->sub_attach) { sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj); if (!sch->sub_kset) { RCU_INIT_POINTER(ops->priv, NULL); kobject_put(&sch->kobj); return ERR_PTR(-ENOMEM); } }#else /* CONFIG_EXT_SUB_SCHED */ ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); if (ret < 0) { RCU_INIT_POINTER(ops->priv, NULL); kobject_put(&sch->kobj); return ERR_PTR(ret); }#endif /* CONFIG_EXT_SUB_SCHED */
/* * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid * drops the ref. After this point, sch owns the ref and any cleanup * runs through scx_sched_free_rcu_work() which puts it. */ sch->arena_map = cmd->arena_map; /* BPF arena is only available on MMU && 64BIT */#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) if (sch->arena_map) sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map);#endif cmd->arena_map = NULL; return sch;
#ifdef CONFIG_EXT_SUB_SCHEDerr_free_lb_resched: free_cpumask_var(sch->bypass_lb_resched_cpumask);#endiferr_free_lb_cpumask: free_cpumask_var(sch->bypass_lb_donee_cpumask);err_stop_helper: kthread_destroy_worker(sch->helper);err_free_pcpu: for_each_possible_cpu(cpu) { if (cpu == bypass_fail_cpu) break; exit_dsq(bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu);err_free_pnode: for_each_node_state(node, N_POSSIBLE) free_pnode(sch->pnode[node]); kfree(sch->pnode);err_free_hash: rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);err_free_ei: free_exit_info(sch->exit_info);err_free_sch: kfree(sch);err_put_cgrp:#ifdef CONFIG_EXT_SUB_SCHED cgroup_put(cgrp);#endif return ERR_PTR(ret);}
static int check_hotplug_seq(struct scx_sched *sch, const struct sched_ext_ops *ops){ unsigned long long global_hotplug_seq;
/* * If a hotplug event has occurred between when a scheduler was * initialized, and when we were able to attach, exit and notify user * space about it. */ if (ops->hotplug_seq) { global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); if (ops->hotplug_seq != global_hotplug_seq) { scx_exit(sch, SCX_EXIT_UNREG_KERN, SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, "expected hotplug seq %llu did not match actual %llu", ops->hotplug_seq, global_hotplug_seq); return -EBUSY; } }
return 0;}
static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops){ /* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the * ops.enqueue() callback isn't implemented. */ if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); return -EINVAL; }
/* * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched * may set it to declare a dependency; reject if the root hasn't * enabled it. */ if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) && !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) { scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it"); return -EINVAL; }
/* * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle * selection policy to be enabled. */ if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) && (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) { scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled"); return -EINVAL; }
/* * cid-form's struct is shorter and doesn't include the cpu_acquire / * cpu_release tail; reading those fields off a cid-form @ops would * run past the BPF allocation. Skip for cid-form. */ if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
/* * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched * attaches through a cid-form-only interface (sub_attach/sub_detach), * and a root that accepts sub-scheds must expose cid-form state to * them. Reject cpu-form schedulers on either side. */ if (!sch->is_cid_type) { if (scx_parent(sch)) { scx_error(sch, "sub-sched requires cid-form struct_ops"); return -EINVAL; } if (ops->sub_attach || ops->sub_detach) { scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops"); return -EINVAL; } }
return 0;}
static void scx_root_enable_workfn(struct kthread_work *work){ struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); struct sched_ext_ops *ops = cmd->ops; struct cgroup *cgrp = root_cgroup(); struct scx_sched *sch; struct scx_task_iter sti; struct task_struct *p; int i, cpu, ret;
mutex_lock(&scx_enable_mutex);
if (scx_enable_state() != SCX_DISABLED) { ret = -EBUSY; goto err_unlock; }
/* * @ops->priv binds @ops to its scx_sched instance. It is set here by * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(), * which runs after scx_root_disable() has dropped scx_enable_mutex. If * it's still non-NULL here, a previous attachment on @ops has not * finished tearing down; proceeding would let the in-flight unreg's * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign. */ if (rcu_access_pointer(ops->priv)) { ret = -EBUSY; goto err_unlock; }
ret = alloc_kick_syncs(); if (ret) goto err_unlock;
if (ops->flags & SCX_OPS_TID_TO_TASK) { ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params); if (ret) goto err_free_ksyncs; }
#ifdef CONFIG_EXT_SUB_SCHED cgroup_get(cgrp);#endif sch = scx_alloc_and_add_sched(cmd, cgrp, NULL); if (IS_ERR(sch)) { ret = PTR_ERR(sch); goto err_free_tid_hash; }
if (sch->is_cid_type) static_branch_enable(&__scx_is_cid_type);
/* * Transition to ENABLING and clear exit info to arm the disable path. * Failure triggers full disabling from here on. */ WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED); WARN_ON_ONCE(scx_root);
atomic_long_set(&scx_nr_rejected, 0);
for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu);
rq->scx.local_dsq.sched = sch; rq->scx.cpuperf_target = SCX_CPUPERF_ONE; }
/* * Keep CPUs stable during enable so that the BPF scheduler can track * online CPUs by watching ->on/offline_cpu() after ->init(). */ cpus_read_lock();
/* * Build the cid mapping before publishing scx_root. The cid kfuncs * dereference the cid arrays unconditionally once scx_prog_sched() * returns non-NULL; the rcu_assign_pointer() below pairs with their * rcu_dereference() to make the populated arrays visible. */ ret = scx_cid_init(sch); if (ret) { cpus_read_unlock(); goto err_disable; }
/* * Make the scheduler instance visible. Must be inside cpus_read_lock(). * See handle_hotplug(). */ rcu_assign_pointer(scx_root, sch);
ret = scx_link_sched(sch); if (ret) { cpus_read_unlock(); goto err_disable; }
scx_idle_enable(ops);
if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { ret = ops_sanitize_err(sch, "init", ret); cpus_read_unlock(); scx_error(sch, "ops.init() failed (%d)", ret); goto err_disable; } sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; }
ret = scx_arena_pool_init(sch); if (ret) { cpus_read_unlock(); goto err_disable; }
ret = scx_set_cmask_scratch_alloc(sch); if (ret) { cpus_read_unlock(); goto err_disable; }
for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op);
ret = check_hotplug_seq(sch, ops); if (ret) { cpus_read_unlock(); goto err_disable; } scx_idle_update_selcpu_topology(ops);
cpus_read_unlock();
ret = validate_ops(sch, ops); if (ret) goto err_disable;
/* * Attach the ext_server bandwidth reservation before anything is * committed so that we can fail the enable if the root domain cannot * accommodate it. The matching fair_server detach is deferred to the * tail of this function, after the switch is fully committed and can no * longer fail. * * On failure, err_disable funnels into scx_root_disable() which * detaches ext_server, so partially-attached state is cleaned up * automatically. */ for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu);
scoped_guard(rq_lock_irqsave, rq) { update_rq_clock(rq); ret = dl_server_attach_bw(&rq->ext_server); } if (ret) { pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n", cpu, ret); goto err_disable; } }
/* * Once __scx_enabled is set, %current can be switched to SCX anytime. * This can lead to stalls as some BPF schedulers (e.g. userspace * scheduling) may not function correctly before all tasks are switched. * Init in bypass mode to guarantee forward progress. */ scx_bypass(sch, true);
for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op);
if (sch->ops.cpu_acquire || sch->ops.cpu_release) sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT;
/* * Lock out forks, cgroup on/offlining and moves before opening the * floodgate so that they don't wander into the operations prematurely. */ percpu_down_write(&scx_fork_rwsem);
WARN_ON_ONCE(scx_init_task_enabled); scx_init_task_enabled = true;
/* flip under fork_rwsem; the iter below covers existing tasks */ if (ops->flags & SCX_OPS_TID_TO_TASK) static_branch_enable(&__scx_tid_to_task_enabled);
/* * Enable ops for every task. Fork is excluded by scx_fork_rwsem * preventing new tasks from being added. No need to exclude tasks * leaving as sched_ext_free() can handle both prepped and enabled * tasks. Prep all tasks first and then enable them with preemption * disabled. * * All cgroups should be initialized before scx_init_task() so that the * BPF scheduler can reliably track each task's cgroup membership from * scx_init_task(). Lock out cgroup on/offlining and task migrations * while tasks are being initialized so that scx_cgroup_can_attach() * never sees uninitialized tasks. */ scx_cgroup_lock(); set_cgroup_sched(sch_cgroup(sch), sch); ret = scx_cgroup_init(sch); if (ret) goto err_disable_unlock_all;
scx_task_iter_start(&sti, NULL); while ((p = scx_task_iter_next_locked(&sti))) { /* * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD * tasks are filtered by scx_task_iter_next_locked(). * sched_ext_dead() removes @p from scx_tasks under the same * lock before put_task_struct_rcu_user() runs, so @p->usage * is guaranteed > 0 here. */ get_task_struct(p);
/* * Set %INIT_BEGIN under the iter's rq lock so that a concurrent * sched_ext_dead() does not call ops.exit_task() on @p while * ops.init_task() is running. If sched_ext_dead() runs before * this store, it has already removed @p from scx_tasks and the * iter won't visit @p; if it runs after, it observes * %INIT_BEGIN and transitions to %DEAD without calling ops, * leaving the post-init recheck below to unwind. */ scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_task_iter_unlock(&sti);
ret = __scx_init_task(sch, p, false);
scx_task_iter_relock(&sti, p);
if (unlikely(ret)) { if (scx_get_task_state(p) != SCX_TASK_DEAD) scx_set_task_state(p, SCX_TASK_NONE); scx_task_iter_stop(&sti); scx_error(sch, "ops.init_task() failed (%d) for %s[%d]", ret, p->comm, p->pid); put_task_struct(p); goto err_disable_unlock_all; }
if (scx_get_task_state(p) == SCX_TASK_DEAD) { /* * sched_ext_dead() observed %INIT_BEGIN and set %DEAD. * ops.exit_task() is owed to the sched __scx_init_task() * ran against; call it now. */ scx_sub_init_cancel_task(sch, p); } else { scx_set_task_state(p, SCX_TASK_INIT); scx_set_task_sched(p, sch); scx_set_task_state(p, SCX_TASK_READY); }
/* * Insert into the tid hash. scx_tasks_lock is held by the iter; * list_empty() guards against sched_ext_dead() having taken @p * off the list while init ran unlocked. */ if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node)) scx_tid_hash_insert(p);
put_task_struct(p); } scx_task_iter_stop(&sti); scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem);
/* * All tasks are READY. It's safe to turn on scx_enabled() and switch * all eligible tasks. */ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); static_branch_enable(&__scx_enabled);
/* * We're fully committed and can't fail. The task READY -> ENABLED * transitions here are synchronized against sched_ext_free() through * scx_tasks_lock. */ percpu_down_write(&scx_fork_rwsem); scx_task_iter_start(&sti, NULL); while ((p = scx_task_iter_next_locked(&sti))) { unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; const struct sched_class *old_class = p->sched_class; const struct sched_class *new_class = scx_setscheduler_class(p);
if (scx_get_task_state(p) != SCX_TASK_READY) continue;
if (old_class != new_class) queue_flags |= DEQUEUE_CLASS;
scoped_guard (sched_change, p, queue_flags) { p->scx.slice = READ_ONCE(sch->slice_dfl); p->sched_class = new_class; } } scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem);
scx_bypass(sch, false);
if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) { WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE); goto err_disable; }
if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) static_branch_enable(&__scx_switched_all);
/* * Detach the fair_server bandwidth reservation now that the switch * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no * task will ever run in the fair class, so give that bandwidth * back to the RT class. The matching ext_server attach already * happened earlier; this only releases bandwidth and cannot fail. * * In partial mode keep fair_server attached. */ if (scx_switched_all()) { for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu);
guard(rq_lock_irqsave)(rq); update_rq_clock(rq); dl_server_detach_bw(&rq->fair_server); } }
pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", sch->ops.name, scx_switched_all() ? "" : " (partial)"); kobject_uevent(&sch->kobj, KOBJ_ADD); mutex_unlock(&scx_enable_mutex);
atomic_long_inc(&scx_enable_seq);
cmd->ret = 0; return;
err_free_tid_hash: if (ops->flags & SCX_OPS_TID_TO_TASK) rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);err_free_ksyncs: free_kick_syncs();err_unlock: mutex_unlock(&scx_enable_mutex); cmd->ret = ret; return;
err_disable_unlock_all: scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); /* we'll soon enter disable path, keep bypass on */err_disable: mutex_unlock(&scx_enable_mutex); /* * Returning an error code here would not pass all the error information * to userspace. Record errno using scx_error() for cases scx_error() * wasn't already invoked and exit indicating success so that the error * is notified through ops.exit() with all the details. * * Flush scx_disable_work to ensure that error is reported before init * completion. sch's base reference will be put by bpf_scx_unreg(). */ scx_error(sch, "scx_root_enable() failed (%d)", ret); scx_flush_disable_work(sch); cmd->ret = 0;}
#ifdef CONFIG_EXT_SUB_SCHED/* verify that a scheduler can be attached to @cgrp and return the parent */static struct scx_sched *find_parent_sched(struct cgroup *cgrp){ struct scx_sched *parent = cgrp->scx_sched; struct scx_sched *pos;
lockdep_assert_held(&scx_sched_lock);
/* can't attach twice to the same cgroup */ if (parent->cgrp == cgrp) return ERR_PTR(-EBUSY);
/* does $parent allow sub-scheds? */ if (!parent->ops.sub_attach) return ERR_PTR(-EOPNOTSUPP);
/* can't insert between $parent and its exiting children */ list_for_each_entry(pos, &parent->children, sibling) if (cgroup_is_descendant(pos->cgrp, cgrp)) return ERR_PTR(-EBUSY);
return parent;}
static bool assert_task_ready_or_enabled(struct task_struct *p){ u32 state = scx_get_task_state(p);
switch (state) { case SCX_TASK_READY: case SCX_TASK_ENABLED: return true; default: WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", state, p->comm, p->pid); return false; }}
static void scx_sub_enable_workfn(struct kthread_work *work){ struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); struct sched_ext_ops *ops = cmd->ops; struct cgroup *cgrp; struct scx_sched *parent, *sch; struct scx_task_iter sti; struct task_struct *p; s32 i, ret;
mutex_lock(&scx_enable_mutex);
if (!scx_enabled()) { ret = -ENODEV; goto out_unlock; }
/* See scx_root_enable_workfn() for the @ops->priv check. */ if (rcu_access_pointer(ops->priv)) { ret = -EBUSY; goto out_unlock; }
cgrp = cgroup_get_from_id(ops->sub_cgroup_id); if (IS_ERR(cgrp)) { ret = PTR_ERR(cgrp); goto out_unlock; }
raw_spin_lock_irq(&scx_sched_lock); parent = find_parent_sched(cgrp); if (IS_ERR(parent)) { raw_spin_unlock_irq(&scx_sched_lock); ret = PTR_ERR(parent); goto out_put_cgrp; } kobject_get(&parent->kobj); raw_spin_unlock_irq(&scx_sched_lock);
/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ sch = scx_alloc_and_add_sched(cmd, cgrp, parent); kobject_put(&parent->kobj); if (IS_ERR(sch)) { ret = PTR_ERR(sch); goto out_unlock; }
ret = scx_link_sched(sch); if (ret) goto err_disable;
if (sch->level >= SCX_SUB_MAX_DEPTH) { scx_error(sch, "max nesting depth %d violated", SCX_SUB_MAX_DEPTH); goto err_disable; }
if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { ret = ops_sanitize_err(sch, "init", ret); scx_error(sch, "ops.init() failed (%d)", ret); goto err_disable; } sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; }
ret = scx_arena_pool_init(sch); if (ret) goto err_disable;
ret = scx_set_cmask_scratch_alloc(sch); if (ret) goto err_disable;
if (validate_ops(sch, ops)) goto err_disable;
struct scx_sub_attach_args sub_attach_args = { .ops = &sch->ops, .cgroup_path = sch->cgrp_path, };
ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, &sub_attach_args); if (ret) { ret = ops_sanitize_err(sch, "sub_attach", ret); scx_error(sch, "parent rejected (%d)", ret); goto err_disable; } sch->sub_attached = true;
scx_bypass(sch, true);
for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op);
percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock();
/* * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. */ set_cgroup_sched(sch_cgroup(sch), sch); if (!(cgrp->self.flags & CSS_ONLINE)) { scx_error(sch, "cgroup is not online"); goto err_unlock_and_disable; }
/* * Initialize tasks for the new child $sch without exiting them for * $parent so that the tasks can always be reverted back to $parent * sched on child init failure. */ WARN_ON_ONCE(scx_enabling_sub_sched); scx_enabling_sub_sched = sch;
scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { struct rq *rq; struct rq_flags rf;
/* * Task iteration may visit the same task twice when racing * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which * finished __scx_init_task() and skip if set. * * A task may exit and get freed between __scx_init_task() * completion and scx_enable_task(). In such cases, * scx_disable_and_exit_task() must exit the task for both the * parent and child scheds. */ if (p->scx.flags & SCX_TASK_SUB_INIT) continue;
/* @p is pinned by the iter; see scx_sub_disable() */ get_task_struct(p);
if (!assert_task_ready_or_enabled(p)) { ret = -EINVAL; goto abort; }
scx_task_iter_unlock(&sti);
/* * As $p is still on $parent, it can't be transitioned to INIT. * Let's worry about task state later. Use __scx_init_task(). */ ret = __scx_init_task(sch, p, false); if (ret) goto abort;
rq = task_rq_lock(p, &rf);
if (scx_get_task_state(p) == SCX_TASK_DEAD) { /* * sched_ext_dead() raced us between __scx_init_task() * and this rq lock and ran exit_task() on $parent (the * sched @p was on at that point), not on @sch. @sch's * just-completed init is owed an exit_task() and we * issue it here. */ scx_sub_init_cancel_task(sch, p); task_rq_unlock(rq, p, &rf); put_task_struct(p); continue; }
p->scx.flags |= SCX_TASK_SUB_INIT; task_rq_unlock(rq, p, &rf);
put_task_struct(p); } scx_task_iter_stop(&sti);
/* * All tasks are prepped. Disable/exit tasks for $parent and enable for * the new @sch. */ scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { /* * Use clearing of %SCX_TASK_SUB_INIT to detect and skip * duplicate iterations. */ if (!(p->scx.flags & SCX_TASK_SUB_INIT)) continue;
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* * $p must be either READY or ENABLED. If ENABLED, * __scx_disabled_and_exit_task() first disables and * makes it READY. However, after exiting $p, it will * leave $p as READY. */ assert_task_ready_or_enabled(p); __scx_disable_and_exit_task(parent, p);
/* * $p is now only initialized for @sch and READY, which * is what we want. Assign it to @sch and enable. */ scx_set_task_sched(p, sch); scx_enable_task(sch, p);
p->scx.flags &= ~SCX_TASK_SUB_INIT; } } scx_task_iter_stop(&sti);
scx_enabling_sub_sched = NULL;
scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem);
scx_bypass(sch, false);
pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); kobject_uevent(&sch->kobj, KOBJ_ADD); ret = 0; goto out_unlock;
out_put_cgrp: cgroup_put(cgrp);out_unlock: mutex_unlock(&scx_enable_mutex); cmd->ret = ret; return;
abort: put_task_struct(p); scx_task_iter_stop(&sti);
/* * Undo __scx_init_task() for tasks we marked. scx_enable_task() never * ran for @sch on them, so calling scx_disable_task() here would invoke * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched * must stay set until SUB_INIT is cleared from every marked task - * scx_disable_and_exit_task() reads it when a task exits concurrently. */ scx_task_iter_start(&sti, sch->cgrp); while ((p = scx_task_iter_next_locked(&sti))) { if (p->scx.flags & SCX_TASK_SUB_INIT) { scx_sub_init_cancel_task(sch, p); p->scx.flags &= ~SCX_TASK_SUB_INIT; } } scx_task_iter_stop(&sti); scx_enabling_sub_sched = NULL;err_unlock_and_disable: /* we'll soon enter disable path, keep bypass on */ scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem);err_disable: mutex_unlock(&scx_enable_mutex); /* * Some enable failures only return an errno (e.g. -ENOMEM from an * allocation) without calling scx_error(). Record it so * scx_flush_disable_work() runs the disable and ops.exit() fires. */ scx_error(sch, "scx_sub_enable() failed (%d)", ret); scx_flush_disable_work(sch); cmd->ret = 0;}
static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, unsigned long action, void *data){ struct cgroup *cgrp = data; struct cgroup *parent = cgroup_parent(cgrp);
if (!cgroup_on_dfl(cgrp)) return NOTIFY_OK;
switch (action) { case CGROUP_LIFETIME_ONLINE: /* inherit ->scx_sched from $parent */ if (parent) rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); break; case CGROUP_LIFETIME_OFFLINE: /* if there is a sched attached, shoot it down */ if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, SCX_ECODE_RSN_CGROUP_OFFLINE, "cgroup %llu going offline", cgroup_id(cgrp)); break; }
return NOTIFY_OK;}
static struct notifier_block scx_cgroup_lifetime_nb = { .notifier_call = scx_cgroup_lifetime_notify,};
static s32 __init scx_cgroup_lifetime_notifier_init(void){ return blocking_notifier_chain_register(&cgroup_lifetime_notifier, &scx_cgroup_lifetime_nb);}core_initcall(scx_cgroup_lifetime_notifier_init);#endif /* CONFIG_EXT_SUB_SCHED */
static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link){ static struct kthread_worker *helper; static DEFINE_MUTEX(helper_mutex);
if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); return -EINVAL; }
if (!READ_ONCE(helper)) { mutex_lock(&helper_mutex); if (!helper) { struct kthread_worker *w = kthread_run_worker(0, "scx_enable_helper"); if (IS_ERR_OR_NULL(w)) { mutex_unlock(&helper_mutex); return -ENOMEM; } sched_set_fifo(w->task); WRITE_ONCE(helper, w); } mutex_unlock(&helper_mutex); }
#ifdef CONFIG_EXT_SUB_SCHED if (cmd->ops->sub_cgroup_id > 1) kthread_init_work(&cmd->work, scx_sub_enable_workfn); else#endif /* CONFIG_EXT_SUB_SCHED */ kthread_init_work(&cmd->work, scx_root_enable_workfn);
kthread_queue_work(READ_ONCE(helper), &cmd->work); kthread_flush_work(&cmd->work); return cmd->ret;}
/******************************************************************************** * bpf_struct_ops plumbing. */#include <linux/bpf_verifier.h>#include <linux/bpf.h>#include <linux/btf.h>
static const struct btf_type *task_struct_type;
static bool bpf_scx_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info){ if (type != BPF_READ) return false; if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) return false; if (off % size != 0) return false;
return btf_ctx_access(off, size, type, prog, info);}
static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size){ const struct btf_type *t;
t = btf_type_by_id(reg->btf, reg->btf_id); if (t == task_struct_type) { /* * COMPAT: Will be removed in v6.23. */ if ((off >= offsetof(struct task_struct, scx.slice) && off + size <= offsetofend(struct task_struct, scx.slice)) || (off >= offsetof(struct task_struct, scx.dsq_vtime) && off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n"); return SCALAR_VALUE; }
if (off >= offsetof(struct task_struct, scx.disallow) && off + size <= offsetofend(struct task_struct, scx.disallow)) return SCALAR_VALUE; }
return -EACCES;}
static const struct bpf_verifier_ops bpf_scx_verifier_ops = { .get_func_proto = bpf_base_func_proto, .is_valid_access = bpf_scx_is_valid_access, .btf_struct_access = bpf_scx_btf_struct_access,};
static int bpf_scx_init_member(const struct btf_type *t, const struct btf_member *member, void *kdata, const void *udata){ const struct sched_ext_ops *uops = udata; struct sched_ext_ops *ops = kdata; u32 moff = __btf_member_bit_offset(t, member) / 8; int ret;
switch (moff) { case offsetof(struct sched_ext_ops, dispatch_max_batch): if (*(u32 *)(udata + moff) > INT_MAX) return -E2BIG; ops->dispatch_max_batch = *(u32 *)(udata + moff); return 1; case offsetof(struct sched_ext_ops, flags): if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) return -EINVAL; ops->flags = *(u64 *)(udata + moff); return 1; case offsetof(struct sched_ext_ops, name): ret = bpf_obj_name_cpy(ops->name, uops->name, sizeof(ops->name)); if (ret < 0) return ret; if (ret == 0) return -EINVAL; return 1; case offsetof(struct sched_ext_ops, timeout_ms): if (msecs_to_jiffies(*(u32 *)(udata + moff)) > SCX_WATCHDOG_MAX_TIMEOUT) return -E2BIG; ops->timeout_ms = *(u32 *)(udata + moff); return 1; case offsetof(struct sched_ext_ops, exit_dump_len): ops->exit_dump_len = *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; return 1; case offsetof(struct sched_ext_ops, hotplug_seq): ops->hotplug_seq = *(u64 *)(udata + moff); return 1;#ifdef CONFIG_EXT_SUB_SCHED case offsetof(struct sched_ext_ops, sub_cgroup_id): ops->sub_cgroup_id = *(u64 *)(udata + moff); return 1;#endif /* CONFIG_EXT_SUB_SCHED */ }
return 0;}
#ifdef CONFIG_EXT_SUB_SCHEDstatic void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog){ struct scx_sched *sch;
guard(rcu)(); sch = scx_prog_sched(prog->aux); if (unlikely(!sch)) return;
scx_error(sch, "dispatch recursion detected");}#endif /* CONFIG_EXT_SUB_SCHED */
static int bpf_scx_check_member(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog){ u32 moff = __btf_member_bit_offset(t, member) / 8;
switch (moff) { case offsetof(struct sched_ext_ops, init_task):#ifdef CONFIG_EXT_GROUP_SCHED case offsetof(struct sched_ext_ops, cgroup_init): case offsetof(struct sched_ext_ops, cgroup_exit): case offsetof(struct sched_ext_ops, cgroup_prep_move):#endif case offsetof(struct sched_ext_ops, cpu_online): case offsetof(struct sched_ext_ops, cpu_offline): case offsetof(struct sched_ext_ops, init): case offsetof(struct sched_ext_ops, exit): case offsetof(struct sched_ext_ops, sub_attach): case offsetof(struct sched_ext_ops, sub_detach): break; default: if (prog->sleepable) return -EINVAL; }
#ifdef CONFIG_EXT_SUB_SCHED /* * Enable private stack for operations that can nest along the * hierarchy. * * XXX - Ideally, we should only do this for scheds that allow * sub-scheds and sub-scheds themselves but I don't know how to access * struct_ops from here. */ switch (moff) { case offsetof(struct sched_ext_ops, dispatch): prog->aux->priv_stack_requested = true; prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch; }#endif /* CONFIG_EXT_SUB_SCHED */
return 0;}
static int bpf_scx_reg(void *kdata, struct bpf_link *link){ struct scx_enable_cmd cmd = { .ops = kdata };
return scx_enable(&cmd, link);}
struct scx_arena_scan { struct bpf_map *arena; int err;};
/* * The verifier enforces one arena per BPF program, so each struct_ops * member prog contributes at most one arena via bpf_prog_arena(). * Require all non-NULL contributions to match. */static int scx_arena_scan_prog(struct bpf_prog *prog, void *data){ struct scx_arena_scan *s = data; struct bpf_map *arena = NULL;
/* arena.o, which defines these, is built only on MMU && 64BIT */#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) arena = bpf_prog_arena(prog);#endif if (!arena) return 0; if (s->arena && s->arena != arena) { s->err = -EINVAL; return 1; } s->arena = arena; return 0;}
static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link){ struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true }; struct scx_arena_scan scan = {}; int ret;
bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan); if (scan.err) { pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n"); return scan.err; } if (!scan.arena) { pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n"); return -EINVAL; }
bpf_map_inc(scan.arena); cmd.arena_map = scan.arena; ret = scx_enable(&cmd, link); if (cmd.arena_map) /* not consumed by scx_alloc_and_add_sched() */ bpf_map_put(cmd.arena_map); return ret;}
static void bpf_scx_unreg(void *kdata, struct bpf_link *link){ struct sched_ext_ops *ops = kdata; struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
scx_disable(sch, SCX_EXIT_UNREG); scx_flush_disable_work(sch); RCU_INIT_POINTER(ops->priv, NULL); kobject_put(&sch->kobj);}
static int bpf_scx_init(struct btf *btf){ task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]);
return 0;}
static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link){ /* * sched_ext does not support updating the actively-loaded BPF * scheduler, as registering a BPF scheduler can always fail if the * scheduler returns an error code for e.g. ops.init(), ops.init_task(), * etc. Similarly, we can always race with unregistration happening * elsewhere, such as with sysrq. */ return -EOPNOTSUPP;}
static int bpf_scx_validate(void *kdata){ return 0;}
static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; }static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {}static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {}static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {}static void sched_ext_ops__tick(struct task_struct *p) {}static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {}static void sched_ext_ops__running(struct task_struct *p) {}static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {}static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {}static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; }static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; }static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {}static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {}static void sched_ext_ops__update_idle(s32 cpu, bool idle) {}static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {}static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {}static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; }static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {}static void sched_ext_ops__enable(struct task_struct *p) {}static void sched_ext_ops__disable(struct task_struct *p) {}#ifdef CONFIG_EXT_GROUP_SCHEDstatic s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; }static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {}static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; }static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {}static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {}static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {}static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {}static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {}#endif /* CONFIG_EXT_GROUP_SCHED */static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; }static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {}static void sched_ext_ops__cpu_online(s32 cpu) {}static void sched_ext_ops__cpu_offline(s32 cpu) {}static s32 sched_ext_ops__init(void) { return -EINVAL; }static void sched_ext_ops__exit(struct scx_exit_info *info) {}static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {}static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {}static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {}
static struct sched_ext_ops __bpf_ops_sched_ext_ops = { .select_cpu = sched_ext_ops__select_cpu, .enqueue = sched_ext_ops__enqueue, .dequeue = sched_ext_ops__dequeue, .dispatch = sched_ext_ops__dispatch, .tick = sched_ext_ops__tick, .runnable = sched_ext_ops__runnable, .running = sched_ext_ops__running, .stopping = sched_ext_ops__stopping, .quiescent = sched_ext_ops__quiescent, .yield = sched_ext_ops__yield, .core_sched_before = sched_ext_ops__core_sched_before, .set_weight = sched_ext_ops__set_weight, .set_cpumask = sched_ext_ops__set_cpumask, .update_idle = sched_ext_ops__update_idle, .cpu_acquire = sched_ext_ops__cpu_acquire, .cpu_release = sched_ext_ops__cpu_release, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, .enable = sched_ext_ops__enable, .disable = sched_ext_ops__disable,#ifdef CONFIG_EXT_GROUP_SCHED .cgroup_init = sched_ext_ops__cgroup_init, .cgroup_exit = sched_ext_ops__cgroup_exit, .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, .cgroup_move = sched_ext_ops__cgroup_move, .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, .cgroup_set_idle = sched_ext_ops__cgroup_set_idle,#endif .sub_attach = sched_ext_ops__sub_attach, .sub_detach = sched_ext_ops__sub_detach, .cpu_online = sched_ext_ops__cpu_online, .cpu_offline = sched_ext_ops__cpu_offline, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, .dump_cpu = sched_ext_ops__dump_cpu, .dump_task = sched_ext_ops__dump_task,};
static struct bpf_struct_ops bpf_sched_ext_ops = { .verifier_ops = &bpf_scx_verifier_ops, .reg = bpf_scx_reg, .unreg = bpf_scx_unreg, .check_member = bpf_scx_check_member, .init_member = bpf_scx_init_member, .init = bpf_scx_init, .update = bpf_scx_update, .validate = bpf_scx_validate, .name = "sched_ext_ops", .owner = THIS_MODULE, .cfi_stubs = &__bpf_ops_sched_ext_ops};
/* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types * identical, only param names differ across structs) are reused; only * set_cmask needs a fresh stub since the second argument type differs. */static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask) {}
static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, .enqueue = sched_ext_ops__enqueue, .dequeue = sched_ext_ops__dequeue, .dispatch = sched_ext_ops__dispatch, .tick = sched_ext_ops__tick, .runnable = sched_ext_ops__runnable, .running = sched_ext_ops__running, .stopping = sched_ext_ops__stopping, .quiescent = sched_ext_ops__quiescent, .yield = sched_ext_ops__yield, .core_sched_before = sched_ext_ops__core_sched_before, .set_weight = sched_ext_ops__set_weight, .set_cmask = sched_ext_ops_cid__set_cmask, .update_idle = sched_ext_ops__update_idle, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, .enable = sched_ext_ops__enable, .disable = sched_ext_ops__disable,#ifdef CONFIG_EXT_GROUP_SCHED .cgroup_init = sched_ext_ops__cgroup_init, .cgroup_exit = sched_ext_ops__cgroup_exit, .cgroup_prep_move = sched_ext_ops__cgroup_prep_move, .cgroup_move = sched_ext_ops__cgroup_move, .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move, .cgroup_set_weight = sched_ext_ops__cgroup_set_weight, .cgroup_set_bandwidth = sched_ext_ops__cgroup_set_bandwidth, .cgroup_set_idle = sched_ext_ops__cgroup_set_idle,#endif .sub_attach = sched_ext_ops__sub_attach, .sub_detach = sched_ext_ops__sub_detach, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init = sched_ext_ops__init, .exit = sched_ext_ops__exit, .dump = sched_ext_ops__dump, .dump_cid = sched_ext_ops__dump_cpu, .dump_task = sched_ext_ops__dump_task,};
/* * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form. * init_member, check_member, reg, unreg, etc. process kdata as the byte block * verified to match by the BUILD_BUG_ON checks in scx_init(). */static struct bpf_struct_ops bpf_sched_ext_ops_cid = { .verifier_ops = &bpf_scx_verifier_ops, .reg = bpf_scx_reg_cid, .unreg = bpf_scx_unreg, .check_member = bpf_scx_check_member, .init_member = bpf_scx_init_member, .init = bpf_scx_init, .update = bpf_scx_update, .validate = bpf_scx_validate, .name = "sched_ext_ops_cid", .owner = THIS_MODULE, .cfi_stubs = &__bpf_ops_sched_ext_ops_cid};
/******************************************************************************** * System integration and init. */
static void sysrq_handle_sched_ext_reset(u8 key){ struct scx_sched *sch;
sch = rcu_dereference(scx_root); if (likely(sch)) scx_disable(sch, SCX_EXIT_SYSRQ); else pr_info("sched_ext: BPF schedulers not loaded\n");}
static const struct sysrq_key_op sysrq_sched_ext_reset_op = { .handler = sysrq_handle_sched_ext_reset, .help_msg = "reset-sched-ext(S)", .action_msg = "Disable sched_ext and revert all tasks to CFS", .enable_mask = SYSRQ_ENABLE_RTNICE,};
static void sysrq_handle_sched_ext_dump(u8 key){ struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .exit_cpu = -1, .reason = "SysRq-D", }; struct scx_sched *sch;
list_for_each_entry_rcu(sch, &scx_sched_all, all) scx_dump_state(sch, &ei, 0, false);}
static const struct sysrq_key_op sysrq_sched_ext_dump_op = { .handler = sysrq_handle_sched_ext_dump, .help_msg = "dump-sched-ext(D)", .action_msg = "Trigger sched_ext debug dump", .enable_mask = SYSRQ_ENABLE_RTNICE,};
static bool can_skip_idle_kick(struct rq *rq){ lockdep_assert_rq_held(rq);
/* * We can skip idle kicking if @rq is going to go through at least one * full SCX scheduling cycle before going idle. Just checking whether * curr is not idle is insufficient because we could be racing * balance_one() trying to pull the next task from a remote rq, which * may fail, and @rq may become idle afterwards. * * The race window is small and we don't and can't guarantee that @rq is * only kicked while idle anyway. Skip only when sure. */ return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE);}
static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs){ struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; const struct sched_class *cur_class; bool should_wait = false; unsigned long flags;
raw_spin_rq_lock_irqsave(rq, flags); cur_class = rq->curr->sched_class;
/* * During CPU hotplug, a CPU may depend on kicking itself to make * forward progress. Allow kicking self regardless of online state. If * @cpu is running a higher class task, we have no control over @cpu. * Skip kicking. */ if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) && !sched_class_above(cur_class, &ext_sched_class)) { if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { if (cur_class == &ext_sched_class) rq->curr->scx.slice = 0; cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); }
if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; should_wait = true; } cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); }
resched_curr(rq); } else { cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); }
raw_spin_rq_unlock_irqrestore(rq, flags);
return should_wait;}
static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq){ struct rq *rq = cpu_rq(cpu); unsigned long flags;
raw_spin_rq_lock_irqsave(rq, flags);
if (!can_skip_idle_kick(rq) && (cpu_online(cpu) || cpu == cpu_of(this_rq))) resched_curr(rq);
raw_spin_rq_unlock_irqrestore(rq, flags);}
static void kick_cpus_irq_workfn(struct irq_work *irq_work){ struct rq *this_rq = this_rq(); struct scx_rq *this_scx = &this_rq->scx; struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs); bool should_wait = false; unsigned long *ksyncs; s32 cpu;
/* can race with free_kick_syncs() during scheduler disable */ if (unlikely(!ksyncs_pcpu)) return;
ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs;
for_each_cpu(cpu, this_scx->cpus_to_kick) { should_wait |= kick_one_cpu(cpu, this_rq, ksyncs); cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); }
for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { kick_one_cpu_if_idle(cpu, this_rq); cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); }
/* * Can't wait in hardirq — kick_sync can't advance, deadlocking if * CPUs wait for each other. Defer to kick_sync_wait_bal_cb(). */ if (should_wait) { raw_spin_rq_lock(this_rq); this_scx->kick_sync_pending = true; resched_curr(this_rq); raw_spin_rq_unlock(this_rq); }}
/** * print_scx_info - print out sched_ext scheduler state * @log_lvl: the log level to use when printing * @p: target task * * If a sched_ext scheduler is enabled, print the name and state of the * scheduler. If @p is on sched_ext, print further information about the task. * * This function can be safely called on any task as long as the task_struct * itself is accessible. While safe, this function isn't synchronized and may * print out mixups or garbages of limited length. */void print_scx_info(const char *log_lvl, struct task_struct *p){ struct scx_sched *sch; enum scx_enable_state state = scx_enable_state(); const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; char runnable_at_buf[22] = "?"; struct sched_class *class; unsigned long runnable_at;
guard(rcu)();
sch = scx_task_sched_rcu(p);
if (!sch) return;
/* * Carefully check if the task was running on sched_ext, and then * carefully copy the time it's been runnable, and its state. */ if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || class != &ext_sched_class) { printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name, scx_enable_state_str[state], all); return; }
if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, sizeof(runnable_at))) scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", jiffies_delta_msecs(runnable_at, jiffies));
/* print everything onto one line to conserve console space */ printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", log_lvl, sch->ops.name, scx_enable_state_str[state], all, runnable_at_buf);}
static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr){ struct scx_sched *sch;
guard(rcu)();
sch = rcu_dereference(scx_root); if (!sch) return NOTIFY_OK;
/* * SCX schedulers often have userspace components which are sometimes * involved in critial scheduling paths. PM operations involve freezing * userspace which can lead to scheduling misbehaviors including stalls. * Let's bypass while PM operations are in progress. */ switch (event) { case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: case PM_RESTORE_PREPARE: scx_bypass(sch, true); break; case PM_POST_HIBERNATION: case PM_POST_SUSPEND: case PM_POST_RESTORE: scx_bypass(sch, false); break; }
return NOTIFY_OK;}
static struct notifier_block scx_pm_notifier = { .notifier_call = scx_pm_handler,};
void __init init_sched_ext_class(void){ s32 cpu, v;
/* * The following is to prevent the compiler from optimizing out the enum * definitions so that BPF scheduler implementations can use them * through the generated vmlinux.h. */ WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | SCX_TG_ONLINE);
scx_idle_init_masks();
for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); int n = cpu_to_node(cpu);
/* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
INIT_LIST_HEAD(&rq->scx.runnable_list); INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n)); BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n)); raw_spin_lock_init(&rq->scx.deferred_reenq_lock); INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals); INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
if (cpu_online(cpu)) cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; }
register_sysrq_key('S', &sysrq_sched_ext_reset_op); register_sysrq_key('D', &sysrq_sched_ext_dump_op); INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
#ifdef CONFIG_EXT_SUB_SCHED BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params));#endif /* CONFIG_EXT_SUB_SCHED */}
/******************************************************************************** * Helpers that can be called from the BPF scheduler. */static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags){ bool is_local = dsq_id == SCX_DSQ_LOCAL || (dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;
if (*enq_flags & SCX_ENQ_IMMED) { if (unlikely(!is_local)) { scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id); return false; } } else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) { *enq_flags |= SCX_ENQ_IMMED; }
return true;}
static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p, u64 dsq_id, u64 *enq_flags){ lockdep_assert_irqs_disabled();
if (unlikely(!p)) { scx_error(sch, "called with NULL task"); return false; }
if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) { scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags); return false; }
/* see SCX_EV_INSERT_NOT_OWNED definition */ if (unlikely(!scx_task_on_sched(sch, p))) { __scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1); return false; }
if (!scx_vet_enq_flags(sch, dsq_id, enq_flags)) return false;
return true;}
static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p, u64 dsq_id, u64 enq_flags){ struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; struct task_struct *ddsp_task;
ddsp_task = __this_cpu_read(direct_dispatch_task); if (ddsp_task) { mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags); return; }
if (unlikely(dspc->cursor >= sch->dsp_max_batch)) { scx_error(sch, "dispatch buffer overflow"); return; }
dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ .task = p, .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, .dsq_id = dsq_id, .enq_flags = enq_flags, };}
__bpf_kfunc_start_defs();
/** * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ * @p: task_struct to insert * @dsq_id: DSQ to insert into * @slice: duration @p can run for in nsecs, 0 to keep the current value * @enq_flags: SCX_ENQ_* * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to * call this function spuriously. Can be called from ops.enqueue(), * ops.select_cpu(), and ops.dispatch(). * * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch * and @p must match the task being enqueued. * * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p * will be directly inserted into the corresponding dispatch queue after * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be * inserted into the local DSQ of the CPU returned by ops.select_cpu(). * @enq_flags are OR'd with the enqueue flags on the enqueue path before the * task is inserted. * * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id * and this function can be called upto ops.dispatch_max_batch times to insert * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the * counter. * * This function doesn't have any locking restrictions and may be called under * BPF locks (in the future when BPF introduces more flexible locking). * * @p is allowed to run for @slice. The scheduling path is triggered on slice * exhaustion. If zero, the current residual slice is maintained. If * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with * scx_bpf_kick_cpu() to trigger scheduling. * * Returns %true on successful insertion, %false on failure. On the root * scheduler, %false return triggers scheduler abort and the caller doesn't need * to check the return value. */__bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch)) return false;
if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false;
if (slice) p->scx.slice = slice; else p->scx.slice = p->scx.slice ?: 1;
scx_dsq_insert_commit(sch, p, dsq_id, enq_flags);
return true;}
/* * COMPAT: Will be removed in v6.23 along with the ___v2 suffix. */__bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags, const struct bpf_prog_aux *aux){ scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);}
static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags){ if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags)) return false;
if (slice) p->scx.slice = slice; else p->scx.slice = p->scx.slice ?: 1;
p->scx.dsq_vtime = vtime;
scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
return true;}
struct scx_bpf_dsq_insert_vtime_args { /* @p can't be packed together as KF_RCU is not transitive */ u64 dsq_id; u64 slice; u64 vtime; u64 enq_flags;};
/** * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion * @p: task_struct to insert * @args: struct containing the rest of the arguments * @args->dsq_id: DSQ to insert into * @args->slice: duration @p can run for in nsecs, 0 to keep the current value * @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ * @args->enq_flags: SCX_ENQ_* * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided * as an inline wrapper in common.bpf.h. * * Insert @p into the vtime priority queue of the DSQ identified by * @args->dsq_id. Tasks queued into the priority queue are ordered by * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert(). * * @args->vtime ordering is according to time_before64() which considers * wrapping. A numerically larger vtime may indicate an earlier position in the * ordering and vice-versa. * * A DSQ can only be used as a FIFO or priority queue at any given time and this * function must not be called on a DSQ which already has one or more FIFO tasks * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and * SCX_DSQ_GLOBAL) cannot be used as priority queues. * * Returns %true on successful insertion, %false on failure. On the root * scheduler, %false return triggers scheduler abort and the caller doesn't need * to check the return value. */__bpf_kfunc bool__scx_bpf_dsq_insert_vtime(struct task_struct *p, struct scx_bpf_dsq_insert_vtime_args *args, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return false;
return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice, args->vtime, args->enq_flags);}
/* * COMPAT: Will be removed in v6.23. */__bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags){ struct scx_sched *sch;
guard(rcu)();
sch = rcu_dereference(scx_root); if (unlikely(!sch)) return;
#ifdef CONFIG_EXT_SUB_SCHED /* * Disallow if any sub-scheds are attached. There is no way to tell * which scheduler called us, just error out @p's scheduler. */ if (unlikely(!list_empty(&sch->children))) { scx_error(scx_task_sched(p), "__scx_bpf_dsq_insert_vtime() must be used"); return; }#endif
scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU)BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_enqueue_dispatch, .filter = scx_kfunc_context_filter,};
static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, struct task_struct *p, u64 dsq_id, u64 enq_flags){ struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; struct scx_sched *sch; struct rq *this_rq, *src_rq, *locked_rq; bool dispatched = false; bool in_balance; unsigned long flags;
/* * The verifier considers an iterator slot initialized on any * KF_ITER_NEW return, so a BPF program may legally reach here after * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL. */ if (unlikely(!src_dsq)) return false;
sch = src_dsq->sched;
if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags)) return false;
/* * If the BPF scheduler keeps calling this function repeatedly, it can * cause similar live-lock conditions as consume_dispatch_q(). */ if (unlikely(READ_ONCE(sch->aborting))) return false;
if (unlikely(!scx_task_on_sched(sch, p))) { scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler", p->comm, p->pid); return false; }
/* * Can be called from either ops.dispatch() locking this_rq() or any * context where no rq lock is held. If latter, lock @p's task_rq which * we'll likely need anyway. */ src_rq = task_rq(p);
local_irq_save(flags); this_rq = this_rq(); in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
if (in_balance) { if (this_rq != src_rq) switch_rq_lock(this_rq, src_rq); } else { raw_spin_rq_lock(src_rq); }
locked_rq = src_rq; raw_spin_lock(&src_dsq->lock);
/* did someone else get to it while we dropped the locks? */ if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) { raw_spin_unlock(&src_dsq->lock); goto out; }
/* @p is still on $src_dsq and stable, determine the destination */ dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p));
/* * Apply vtime and slice updates before moving so that the new time is * visible before inserting into $dst_dsq. @p is still on $src_dsq but * this is safe as we're locking it. */ if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) p->scx.dsq_vtime = kit->vtime; if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) p->scx.slice = kit->slice;
/* execute move */ locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true;out: if (in_balance) { if (this_rq != locked_rq) switch_rq_lock(locked_rq, this_rq); } else { raw_spin_rq_unlock_irqrestore(locked_rq, flags); }
kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME); return dispatched;}
__bpf_kfunc_start_defs();
/** * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Can only be called from ops.dispatch(). */__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return 0;
return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor);}
/** * scx_bpf_dispatch_cancel - Cancel the latest dispatch * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Cancel the latest dispatch. Can be called multiple times to cancel further * dispatches. Can only be called from ops.dispatch(). */__bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct scx_dsp_ctx *dspc;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return;
dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
if (dspc->cursor > 0) dspc->cursor--; else scx_error(sch, "dispatch buffer underflow");}
/** * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ * @dsq_id: DSQ to move task from. Must be a user-created DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * @enq_flags: %SCX_ENQ_* * * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's * local DSQ for execution with @enq_flags applied. Can only be called from * ops.dispatch(). * * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as * sources. Local DSQs support reenqueueing (a task can be picked up for * execution, dequeued for property changes, or reenqueued), but the BPF * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL * is similar but also doesn't support reenqueueing, as it maps to multiple * per-node DSQs making the scope difficult to define; this may change in the * future. * * This function flushes the in-flight dispatches from scx_bpf_dsq_insert() * before trying to move from the specified DSQ. It may also grab rq locks and * thus can't be called under any BPF locks. * * Returns %true if a task has been moved, %false if there isn't any task to * move. */__bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, const struct bpf_prog_aux *aux){ struct scx_dispatch_q *dsq; struct scx_sched *sch; struct scx_dsp_ctx *dspc;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return false;
if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags)) return false;
dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
flush_dispatch_buf(sch, dspc->rq);
dsq = find_user_dsq(sch, dsq_id); if (unlikely(!dsq)) { scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id); return false; }
if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { /* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched * tasks. Bump nr_tasks to tell balance_one() to retry on empty * local DSQ. */ dspc->nr_tasks++; return true; } else { return false; }}
/* * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future. */__bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux){ return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux);}
/** * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs * @it__iter: DSQ iterator in progress * @slice: duration the moved task can run for in nsecs * * Override the slice of the next task that will be moved from @it__iter using * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous * slice duration is kept. */__bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter, u64 slice){ struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
kit->slice = slice; kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE;}
/** * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs * @it__iter: DSQ iterator in progress * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ * * Override the vtime of the next task that will be moved from @it__iter using * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the * override is ignored and cleared. */__bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter, u64 vtime){ struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
kit->vtime = vtime; kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME;}
/** * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ * @it__iter: DSQ iterator in progress * @p: task to transfer * @dsq_id: DSQ to move @p to * @enq_flags: SCX_ENQ_* * * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can * be the destination. * * For the transfer to be successful, @p must still be on the DSQ and have been * queued before the DSQ iteration started. This function doesn't care whether * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have * been queued before the iteration started. * * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update. * * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq * lock (e.g. BPF timers or SYSCALL programs). * * Returns %true if @p has been consumed, %false if @p had already been * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local * DSQ. */__bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags){ return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, p, dsq_id, enq_flags);}
/** * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ * @it__iter: DSQ iterator in progress * @p: task to transfer * @dsq_id: DSQ to move @p to * @enq_flags: SCX_ENQ_* * * Transfer @p which is on the DSQ currently iterated by @it__iter to the * priority queue of the DSQ specified by @dsq_id. The destination must be a * user DSQ as only user DSQs support priority queue. * * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice() * and scx_bpf_dsq_move_set_vtime() to update. * * All other aspects are identical to scx_bpf_dsq_move(). See * scx_bpf_dsq_insert_vtime() for more information on @vtime. */__bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags){ return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);}
#ifdef CONFIG_EXT_SUB_SCHED/** * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler * @cgroup_id: cgroup ID of the child scheduler to dispatch * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Allows a parent scheduler to trigger dispatching on one of its direct * child schedulers. The child scheduler runs its dispatch operation to * move tasks from dispatch queues to the local runqueue. * * Returns: true on success, false if cgroup_id is invalid, not a direct * child, or caller lacks dispatch permission. */__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux){ struct rq *this_rq = this_rq(); struct scx_sched *parent, *child;
guard(rcu)(); parent = scx_prog_sched(aux); if (unlikely(!parent)) return false;
child = scx_find_sub_sched(cgroup_id);
if (unlikely(!child)) return false;
if (unlikely(scx_parent(child) != parent)) { scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", cgroup_id); return false; }
return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, true);}#endif /* CONFIG_EXT_SUB_SCHED */
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_dispatch)BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS)/* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)#ifdef CONFIG_EXT_SUB_SCHEDBTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS)#endifBTF_KFUNCS_END(scx_kfunc_ids_dispatch)
static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_dispatch, .filter = scx_kfunc_context_filter,};
__bpf_kfunc_start_defs();
/** * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Iterate over all of the tasks currently enqueued on the local DSQ of the * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of * processed tasks. Can only be called from ops.cpu_release(). */__bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct rq *rq;
guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch)) return 0;
rq = cpu_rq(smp_processor_id()); lockdep_assert_rq_held(rq);
return reenq_local(sch, rq, SCX_REENQ_ANY);}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_cpu_release)BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS)BTF_KFUNCS_END(scx_kfunc_ids_cpu_release)
static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_cpu_release, .filter = scx_kfunc_context_filter,};
__bpf_kfunc_start_defs();
/** * scx_bpf_create_dsq - Create a custom DSQ * @dsq_id: DSQ to create * @node: NUMA node to allocate from * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. */__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux){ struct scx_dispatch_q *dsq; struct scx_sched *sch; s32 ret;
if (unlikely(node >= (int)nr_node_ids || (node < 0 && node != NUMA_NO_NODE))) return -EINVAL;
if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) return -EINVAL;
dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); if (!dsq) return -ENOMEM;
/* * init_dsq() must be called in GFP_KERNEL context. Init it with NULL * @sch and update afterwards. */ ret = init_dsq(dsq, dsq_id, NULL); if (ret) { kfree(dsq); return ret; }
rcu_read_lock();
sch = scx_prog_sched(aux); if (sch) { dsq->sched = sch; ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node, dsq_hash_params); } else { ret = -ENODEV; }
rcu_read_unlock(); if (ret) { exit_dsq(dsq); kfree(dsq); } return ret;}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_unlocked)BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE)/* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)/* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU)BTF_KFUNCS_END(scx_kfunc_ids_unlocked)
static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_unlocked, .filter = scx_kfunc_context_filter,};
__bpf_kfunc_start_defs();
/** * scx_bpf_task_set_slice - Set task's time slice * @p: task of interest * @slice: time slice to set in nsecs * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Set @p's time slice to @slice. Returns %true on success, %false if the * calling scheduler doesn't have authority over @p. */__bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) return false;
p->scx.slice = slice; return true;}
/** * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering * @p: task of interest * @vtime: virtual time to set * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Set @p's virtual time to @vtime. Returns %true on success, %false if the * calling scheduler doesn't have authority over @p. */__bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) return false;
p->scx.dsq_vtime = vtime; return true;}
static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags){ struct rq *this_rq; unsigned long irq_flags;
local_irq_save(irq_flags);
this_rq = this_rq();
/* * While bypassing for PM ops, IRQ handling may not be online which can * lead to irq_work_queue() malfunction such as infinite busy wait for * IRQ status update. Suppress kicking. */ if (scx_bypassing(sch, cpu_of(this_rq))) goto out;
/* * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting * rq locks. We can probably be smarter and avoid bouncing if called * from ops which don't hold a rq lock. */ if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu);
if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
if (raw_spin_rq_trylock(target_rq)) { if (can_skip_idle_kick(target_rq)) { raw_spin_rq_unlock(target_rq); goto out; } raw_spin_rq_unlock(target_rq); } cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); } else { cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick);
if (flags & SCX_KICK_PREEMPT) cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); if (flags & SCX_KICK_WAIT) cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); }
irq_work_queue(&this_rq->scx.kick_cpus_irq_work);out: local_irq_restore(irq_flags);}
/** * scx_bpf_kick_cpu - Trigger reschedule on a CPU * @cpu: cpu to kick * @flags: %SCX_KICK_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or * trigger rescheduling on a busy CPU. This can be called from any online * scx_ops operation and the actual kicking is performed asynchronously through * an irq work. */__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)(); sch = scx_prog_sched(aux); if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) scx_kick_cpu(sch, cpu, flags);}
/** * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid * @cid: cid to kick * @flags: %SCX_KICK_* flags * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success, * -errno otherwise. */__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux){ struct scx_sched *sch; s32 cpu;
guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch)) return -ENODEV; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return cpu; scx_kick_cpu(sch, cpu, flags); return 0;}
/** * scx_bpf_dsq_nr_queued - Return the number of queued tasks * @dsq_id: id of the DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Return the number of tasks in the DSQ matching @dsq_id. If not found, * -%ENOENT is returned. */__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct scx_dispatch_q *dsq; s32 ret;
preempt_disable();
sch = scx_prog_sched(aux); if (unlikely(!sch)) { ret = -ENODEV; goto out; }
if (dsq_id == SCX_DSQ_LOCAL) { ret = READ_ONCE(this_rq()->scx.local_dsq.nr); goto out; } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
if (scx_cpu_valid(sch, cpu, NULL)) { ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); goto out; } } else { dsq = find_user_dsq(sch, dsq_id); if (dsq) { ret = READ_ONCE(dsq->nr); goto out; } } ret = -ENOENT;out: preempt_enable(); return ret;}
/** * scx_bpf_destroy_dsq - Destroy a custom DSQ * @dsq_id: DSQ to destroy * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is * empty and no further tasks are dispatched to it. Ignored if called on a DSQ * which doesn't exist. Can be called from any online scx_ops operations. */__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)(); sch = scx_prog_sched(aux); if (sch) destroy_dsq(sch, dsq_id);}
/** * bpf_iter_scx_dsq_new - Create a DSQ iterator * @it: iterator to initialize * @dsq_id: DSQ to iterate * @flags: %SCX_DSQ_ITER_* * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Initialize BPF iterator @it which can be used with bpf_for_each() to walk * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes * tasks which are already queued when this function is invoked. */__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags, const struct bpf_prog_aux *aux){ struct bpf_iter_scx_dsq_kern *kit = (void *)it; struct scx_sched *sch;
BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > sizeof(struct bpf_iter_scx_dsq)); BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != __alignof__(struct bpf_iter_scx_dsq)); BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1));
/* * next() and destroy() will be called regardless of the return value. * Always clear $kit->dsq. */ kit->dsq = NULL;
sch = scx_prog_sched(aux); if (unlikely(!sch)) return -ENODEV;
if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL;
kit->dsq = find_user_dsq(sch, dsq_id); if (!kit->dsq) return -ENOENT;
kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags);
return 0;}
/** * bpf_iter_scx_dsq_next - Progress a DSQ iterator * @it: iterator to progress * * Return the next task. See bpf_iter_scx_dsq_new(). */__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it){ struct bpf_iter_scx_dsq_kern *kit = (void *)it;
if (!kit->dsq) return NULL;
guard(raw_spinlock_irqsave)(&kit->dsq->lock);
return nldsq_cursor_next_task(&kit->cursor, kit->dsq);}
/** * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * * Undo scx_iter_scx_dsq_new(). */__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it){ struct bpf_iter_scx_dsq_kern *kit = (void *)it;
if (!kit->dsq) return;
if (!list_empty(&kit->cursor.node)) { unsigned long flags;
raw_spin_lock_irqsave(&kit->dsq->lock, flags); list_del_init(&kit->cursor.node); raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); } kit->dsq = NULL;}
/** * scx_bpf_dsq_peek - Lockless peek at the first element. * @dsq_id: DSQ to examine. * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Read the first element in the DSQ. This is semantically equivalent to using * the DSQ iterator, but is lockfree. Of course, like any lockless operation, * this provides only a point-in-time snapshot, and the contents may change * by the time any subsequent locking operation reads the queue. * * Returns the pointer, or NULL indicates an empty queue OR internal error. */__bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct scx_dispatch_q *dsq;
sch = scx_prog_sched(aux); if (unlikely(!sch)) return NULL;
if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) { scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id); return NULL; }
dsq = find_user_dsq(sch, dsq_id); if (unlikely(!dsq)) { scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id); return NULL; }
return rcu_dereference(dsq->first_task);}
/** * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ * @dsq_id: DSQ to re-enqueue * @reenq_flags: %SCX_RENQ_* * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Iterate over all of the tasks currently enqueued on the DSQ identified by * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are * supported: * * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) * - User DSQs * * Re-enqueues are performed asynchronously. Can be called from anywhere. */__bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct scx_dispatch_q *dsq;
guard(preempt)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return;
if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) { scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags); return; }
/* not specifying any filter bits is the same as %SCX_REENQ_ANY */ if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY;
dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id()); schedule_dsq_reenq(sch, dsq, reenq_flags, scx_locked_rq());}
/** * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Iterate over all of the tasks currently enqueued on the local DSQ of the * caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from * anywhere. * * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the * future. */__bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux){ scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux);}
__bpf_kfunc_end_defs();
__printf(5, 0)static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf, size_t line_size, char *fmt, unsigned long long *data, u32 data__sz){ struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; s32 ret;
if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || (data__sz && !data)) { scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz); return -EINVAL; }
ret = copy_from_kernel_nofault(data_buf, data, data__sz); if (ret < 0) { scx_error(sch, "failed to read data fields (%d)", ret); return ret; }
ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, &bprintf_data); if (ret < 0) { scx_error(sch, "format preparation failed (%d)", ret); return ret; }
ret = bstr_printf(line_buf, line_size, fmt, bprintf_data.bin_args); bpf_bprintf_cleanup(&bprintf_data); if (ret < 0) { scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz); return ret; }
return ret;}
__printf(3, 0)static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf, char *fmt, unsigned long long *data, u32 data__sz){ return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line), fmt, data, data__sz);}
__bpf_kfunc_start_defs();
/** * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. * @exit_code: Exit value to pass to user space via struct scx_exit_info. * @fmt: error message format string * @data: format string parameters packaged using ___bpf_fill() macro * @data__sz: @data len, must end in '__sz' for the verifier * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops * disabling. */__printf(2, 0)__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux){ struct scx_sched *sch; unsigned long flags;
raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); sch = scx_prog_sched(aux); if (likely(sch) && bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line); raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);}
/** * scx_bpf_error_bstr - Indicate fatal error * @fmt: error message format string * @data: format string parameters packaged using ___bpf_fill() macro * @data__sz: @data len, must end in '__sz' for the verifier * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Indicate that the BPF scheduler encountered a fatal error and initiate ops * disabling. */__printf(1, 0)__bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux){ struct scx_sched *sch; unsigned long flags;
raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); sch = scx_prog_sched(aux); if (likely(sch) && bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0) scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line); raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);}
/** * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler * @fmt: format string * @data: format string parameters packaged using ___bpf_fill() macro * @data__sz: @data len, must end in '__sz' for the verifier * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and * dump_task() to generate extra debug dump specific to the BPF scheduler. * * The extra dump may be multiple lines. A single line may be split over * multiple calls. The last line is automatically terminated. */__printf(1, 0)__bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data__sz, const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct scx_dump_data *dd = &scx_dump_data; struct scx_bstr_buf *buf = &dd->buf; s32 ret;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return;
if (raw_smp_processor_id() != dd->cpu) { scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends"); return; }
/* append the formatted string to the line buf */ ret = __bstr_format(sch, buf->data, buf->line + dd->cursor, sizeof(buf->line) - dd->cursor, fmt, data, data__sz); if (ret < 0) { dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", dd->prefix, fmt, data, data__sz, ret); return; }
dd->cursor += ret; dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line));
if (!dd->cursor) return;
/* * If the line buf overflowed or ends in a newline, flush it into the * dump. This is to allow the caller to generate a single line over * multiple calls. As ops_dump_flush() can also handle multiple lines in * the line buf, the only case which can lead to an unexpected * truncation is when the caller keeps generating newlines in the middle * instead of the end consecutively. Don't do that. */ if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') ops_dump_flush();}
/** * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Return the maximum relative capacity of @cpu in relation to the most * performant CPU in the system. The return value is in the range [1, * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). */__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) return arch_scale_cpu_capacity(cpu); else return SCX_CPUPERF_ONE;}
/** * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid * @cid: cid of the CPU to query * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * cid-addressed equivalent of scx_bpf_cpuperf_cap(). */__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux){ struct scx_sched *sch; s32 cpu;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return SCX_CPUPERF_ONE; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return SCX_CPUPERF_ONE; return arch_scale_cpu_capacity(cpu);}
/** * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Return the current relative performance of @cpu in relation to its maximum. * The return value is in the range [1, %SCX_CPUPERF_ONE]. * * The current performance level of a CPU in relation to the maximum performance * available in the system can be calculated as follows: * * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE * * The result is in the range [1, %SCX_CPUPERF_ONE]. */__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (likely(sch) && scx_cpu_valid(sch, cpu, NULL)) return arch_scale_freq_capacity(cpu); else return SCX_CPUPERF_ONE;}
/** * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid * @cid: cid of the CPU to query * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * cid-addressed equivalent of scx_bpf_cpuperf_cur(). */__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux){ struct scx_sched *sch; s32 cpu;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return SCX_CPUPERF_ONE; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return SCX_CPUPERF_ONE; return arch_scale_freq_capacity(cpu);}
/** * scx_bpf_cpuperf_set - Set the relative performance target of a CPU * @cpu: CPU of interest * @perf: target performance level [0, %SCX_CPUPERF_ONE] * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Set the target performance level of @cpu to @perf. @perf is in linear * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the * schedutil cpufreq governor chooses the target frequency. * * The actual performance level chosen, CPU grouping, and the overhead and * latency of the operations are dependent on the hardware and cpufreq driver in * use. Consult hardware and cpufreq documentation for more information. The * current performance level can be monitored using scx_bpf_cpuperf_cur(). */__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return;
if (unlikely(perf > SCX_CPUPERF_ONE)) { scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu); return; }
if (scx_cpu_valid(sch, cpu, NULL)) { struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq(); struct rq_flags rf;
/* * When called with an rq lock held, restrict the operation * to the corresponding CPU to prevent ABBA deadlocks. */ if (locked_rq && rq != locked_rq) { scx_error(sch, "Invalid target CPU %d", cpu); return; }
/* * If no rq lock is held, allow to operate on any CPU by * acquiring the corresponding rq lock. */ if (!locked_rq) { rq_lock_irqsave(rq, &rf); update_rq_clock(rq); }
rq->scx.cpuperf_target = perf; cpufreq_update_util(rq, 0);
if (!locked_rq) rq_unlock_irqrestore(rq, &rf); }}
/** * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid * @cid: cid of the CPU to target * @perf: target performance level [0, %SCX_CPUPERF_ONE] * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * cid-addressed equivalent of scx_bpf_cpuperf_set(). */__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf, const struct bpf_prog_aux *aux){ struct scx_sched *sch; s32 cpu;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return; scx_bpf_cpuperf_set(cpu, perf, aux);}
/** * scx_bpf_nr_node_ids - Return the number of possible node IDs * * All valid node IDs in the system are smaller than the returned value. */__bpf_kfunc u32 scx_bpf_nr_node_ids(void){ return nr_node_ids;}
/** * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs * * All valid CPU IDs in the system are smaller than the returned value. */__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void){ return nr_cpu_ids;}
/** * scx_bpf_nr_cids - Return the size of the cid space * * Equals num_possible_cpus(). All valid cids are in [0, return value). */__bpf_kfunc u32 scx_bpf_nr_cids(void){ return num_possible_cpus();}
/** * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space * * Return num_online_cpus(). The standard model restarts the scheduler on * hotplug, which lets schedulers treat [0, nr_online_cids) as the online * range. Schedulers that prefer to handle hotplug without a restart should * install a custom mapping via scx_bpf_cid_override() and track onlining * through the ops.cid_online / ops.cid_offline callbacks. */__bpf_kfunc u32 scx_bpf_nr_online_cids(void){ return num_online_cpus();}
/** * scx_bpf_this_cid - Return the cid of the CPU this program is running on * * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs. * The current cpu is trivially valid, so this is just a table lookup. Return * -EINVAL if called from a non-SCX program before any scheduler has ever * been enabled (the cid table is still unallocated at that point). */__bpf_kfunc s32 scx_bpf_this_cid(void){ s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl);
if (!tbl) return -EINVAL; return tbl[raw_smp_processor_id()];}
/** * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask */__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void){ return cpu_possible_mask;}
/** * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask */__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void){ return cpu_online_mask;}
/** * scx_bpf_put_cpumask - Release a possible/online cpumask * @cpumask: cpumask to release */__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask){ /* * Empty function body because we aren't actually acquiring or releasing * a reference to a global cpumask, which is read-only in the caller and * is never released. The acquire / release semantics here are just used * to make the cpumask is a trusted pointer in the caller. */}
/** * scx_bpf_task_running - Is task currently running? * @p: task of interest */__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p){ return task_rq(p)->curr == p;}
/** * scx_bpf_task_cpu - CPU a task is currently associated with * @p: task of interest */__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p){ return task_cpu(p);}
/** * scx_bpf_task_cid - cid a task is currently associated with * @p: task of interest * * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a * valid cpu, so this is just a table lookup. Return -EINVAL if called from * a non-SCX program before any scheduler has ever been enabled. */__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p){ s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl);
if (!tbl) return -EINVAL; return tbl[task_cpu(p)];}
/** * scx_bpf_cpu_rq - Fetch the rq of a CPU * @cpu: CPU of the rq * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs */__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return NULL;
if (!scx_cpu_valid(sch, cpu, NULL)) return NULL;
if (!sch->warned_deprecated_rq) { printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " "use scx_bpf_locked_rq() when holding rq lock " "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); sch->warned_deprecated_rq = true; }
return cpu_rq(cpu);}
/** * scx_bpf_locked_rq - Return the rq currently locked by SCX * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Returns the rq if a rq lock is currently held by SCX. * Otherwise emits an error and returns NULL. */__bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux){ struct scx_sched *sch; struct rq *rq;
guard(preempt)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return NULL;
rq = scx_locked_rq(); if (!rq) { scx_error(sch, "accessing rq without holding rq lock"); return NULL; }
return rq;}
/** * scx_bpf_cpu_curr - Return remote CPU's curr task * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * Callers must hold RCU read lock (KF_RCU). */__bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux){ struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return NULL;
if (!scx_cpu_valid(sch, cpu, NULL)) return NULL;
return rcu_dereference(cpu_rq(cpu)->curr);}
/** * scx_bpf_cid_curr - Return the curr task on the CPU at @cid * @cid: cid of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU * read lock (KF_RCU). */__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux){ struct scx_sched *sch; s32 cpu;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) return NULL; cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return NULL; return rcu_dereference(cpu_rq(cpu)->curr);}
/** * scx_bpf_tid_to_task - Look up a task by its scx tid * @tid: task ID previously read from p->scx.tid * * Returns the task with the given tid, or NULL if no such task exists. The * returned pointer is valid until the end of the current RCU read section * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root * scheduler; otherwise an error is raised and NULL returned. */__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid){ struct sched_ext_entity *scx;
if (!scx_tid_to_task_enabled()) { struct scx_sched *sch = rcu_dereference(scx_root);
if (sch) scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK"); return NULL; }
scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params); if (!scx) return NULL;
return container_of(scx, struct task_struct, scx);}
/** * scx_bpf_now - Returns a high-performance monotonically non-decreasing * clock for the current CPU. The clock returned is in nanoseconds. * * It provides the following properties: * * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently * to account for execution time and track tasks' runtime properties. * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which * eventually reads a hardware timestamp counter -- is neither performant nor * scalable. scx_bpf_now() aims to provide a high-performance clock by * using the rq clock in the scheduler core whenever possible. * * 2) High enough resolution for the BPF scheduler use cases: In most BPF * scheduler use cases, the required clock resolution is lower than the most * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically * uses the rq clock in the scheduler core whenever it is valid. It considers * that the rq clock is valid from the time the rq clock is updated * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). * * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() * guarantees the clock never goes backward when comparing them in the same * CPU. On the other hand, when comparing clocks in different CPUs, there * is no such guarantee -- the clock can go backward. It provides a * monotonically *non-decreasing* clock so that it would provide the same * clock values in two different scx_bpf_now() calls in the same CPU * during the same period of when the rq clock is valid. */__bpf_kfunc u64 scx_bpf_now(void){ struct rq *rq; u64 clock;
preempt_disable();
rq = this_rq(); if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { /* * If the rq clock is valid, use the cached rq clock. * * Note that scx_bpf_now() is re-entrant between a process * context and an interrupt context (e.g., timer interrupt). * However, we don't need to consider the race between them * because such race is not observable from a caller. */ clock = READ_ONCE(rq->scx.clock); } else { /* * Otherwise, return a fresh rq clock. * * The rq clock is updated outside of the rq lock. * In this case, keep the updated rq clock invalid so the next * kfunc call outside the rq lock gets a fresh rq clock. */ clock = sched_clock_cpu(cpu_of(rq)); }
preempt_enable();
return clock;}
static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events){ struct scx_event_stats *e_cpu; int cpu;
/* Aggregate per-CPU event counters into @events. */ memset(events, 0, sizeof(*events)); for_each_possible_cpu(cpu) { e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); }}
/* * scx_bpf_events - Get a system-wide event counter to * @events: output buffer from a BPF program * @events__sz: @events len, must end in '__sz'' for the verifier */__bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, size_t events__sz){ struct scx_sched *sch; struct scx_event_stats e_sys;
rcu_read_lock(); sch = rcu_dereference(scx_root); if (sch) scx_read_events(sch, &e_sys); else memset(&e_sys, 0, sizeof(e_sys)); rcu_read_unlock();
/* * We cannot entirely trust a BPF-provided size since a BPF program * might be compiled against a different vmlinux.h, of which * scx_event_stats would be larger (a newer vmlinux.h) or smaller * (an older vmlinux.h). Hence, we use the smaller size to avoid * memory corruption. */ events__sz = min(events__sz, sizeof(*events)); memcpy(events, &e_sys, events__sz);}
#ifdef CONFIG_CGROUP_SCHED/** * scx_bpf_task_cgroup - Return the sched cgroup of a task * @p: task of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with * from the scheduler's POV. SCX operations should use this function to * determine @p's current cgroup as, unlike following @p->cgroups, * @p->sched_task_group is stable for the duration of the SCX op. See * SCX_CALL_OP_TASK() for details. */__bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, const struct bpf_prog_aux *aux){ struct task_group *tg = p->sched_task_group; struct cgroup *cgrp = &cgrp_dfl_root.cgrp; struct scx_sched *sch;
guard(rcu)();
sch = scx_prog_sched(aux); if (unlikely(!sch)) goto out;
if (!scx_kf_arg_task_ok(sch, p)) goto out;
cgrp = tg_cgrp(tg);
out: cgroup_get(cgrp); return cgrp;}#endif /* CONFIG_CGROUP_SCHED */
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(scx_kfunc_ids_any)BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL)BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED)BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)BTF_ID_FLAGS(func, scx_bpf_nr_cids)BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)BTF_ID_FLAGS(func, scx_bpf_this_cid)BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL)BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED)BTF_ID_FLAGS(func, scx_bpf_now)BTF_ID_FLAGS(func, scx_bpf_events)#ifdef CONFIG_CGROUP_SCHEDBTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE)#endifBTF_KFUNCS_END(scx_kfunc_ids_any)
static const struct btf_kfunc_id_set scx_kfunc_set_any = { .owner = THIS_MODULE, .set = &scx_kfunc_ids_any, .filter = scx_kfunc_context_filter,};
/* * cpu-form kfuncs that are forbidden from cid-form schedulers * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must * use the cid-form alternative (cid/cmask kfuncs). * * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter * tests this set independently and rejects matches before the per-op * allow-list check runs. * * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and * intersects flags across duplicate entries, so each entry must carry the * same flags as the kfunc's primary declaration; otherwise the flags get * dropped globally. */BTF_KFUNCS_START(scx_kfunc_ids_cpu_only)BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS)BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU)BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)BTF_KFUNCS_END(scx_kfunc_ids_cpu_only)
/* * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc * group; an op may permit zero or more groups, with the union expressed in * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter()) * consults this table to decide whether a context-sensitive kfunc is callable * from a given SCX op. */enum scx_kf_allow_flags { SCX_KF_ALLOW_UNLOCKED = 1 << 0, SCX_KF_ALLOW_INIT = 1 << 1, SCX_KF_ALLOW_CPU_RELEASE = 1 << 2, SCX_KF_ALLOW_DISPATCH = 1 << 3, SCX_KF_ALLOW_ENQUEUE = 1 << 4, SCX_KF_ALLOW_SELECT_CPU = 1 << 5,};
/* * Map each SCX op to the union of kfunc groups it permits, indexed by * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not * context-sensitive. */static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(select_cpu)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, [SCX_OP_IDX(enqueue)] = SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE, [SCX_OP_IDX(dispatch)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_release)] = SCX_KF_ALLOW_CPU_RELEASE, [SCX_OP_IDX(init_task)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(dump)] = SCX_KF_ALLOW_UNLOCKED,#ifdef CONFIG_EXT_GROUP_SCHED [SCX_OP_IDX(cgroup_init)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cgroup_exit)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cgroup_prep_move)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cgroup_set_weight)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cgroup_set_idle)] = SCX_KF_ALLOW_UNLOCKED,#endif /* CONFIG_EXT_GROUP_SCHED */ [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(init)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT, [SCX_OP_IDX(exit)] = SCX_KF_ALLOW_UNLOCKED,};
/* * Verifier-time filter for SCX kfuncs. Registered via the .filter field on * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by * falling through to "allow" when none of the SCX sets contain the kfunc. */int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id){ bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id); bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id); bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id); bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id); bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id); bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id); bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id); bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); u32 moff, flags;
/* Not an SCX kfunc - allow. */ if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch || in_cpu_release || in_idle || in_any)) return 0;
/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */ if (prog->type == BPF_PROG_TYPE_SYSCALL) return (in_unlocked || in_select_cpu || in_idle || in_any) ? 0 : -EACCES;
if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) return (in_any || in_idle) ? 0 : -EACCES;
/* * add_subprog_and_kfunc() collects all kfunc calls, including dead code * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set; * do_check_main() re-runs the filter with st_ops set and enforces the * actual restrictions. */ if (!prog->aux->st_ops) return 0;
/* * Non-SCX struct_ops: SCX kfuncs are not permitted. * * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid * (cid-form) are valid SCX struct_ops. Member offsets match between * the two (verified by BUILD_BUG_ON in scx_init()), so the shared * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to * both. */ if (prog->aux->st_ops != &bpf_sched_ext_ops && prog->aux->st_ops != &bpf_sched_ext_ops_cid) return -EACCES;
/* * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both * small s32s and trivially confused, so cpu-only kfuncs are rejected at * load time. The reverse (cpu-form calling cid-form kfuncs) is * intentionally permissive to ease gradual cpumask -> cid migration. */ if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only) return -EACCES;
/* SCX struct_ops: check the per-op allow list. */ if (in_any || in_idle) return 0;
moff = prog->aux->attach_st_ops_member_off; flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];
if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked) return 0; if ((flags & SCX_KF_ALLOW_INIT) && in_init) return 0; if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release) return 0; if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch) return 0; if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue) return 0; if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu) return 0;
return -EACCES;}
static int __init scx_init(void){ int ret;
/* * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv. * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets * matching for the shared fields. Catch any drift at boot. */#define CID_OFFSET_MATCH(cpu_field, cid_field) \ BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) != \ offsetof(struct sched_ext_ops_cid, cid_field)) /* data fields used by bpf_scx_init_member() */ CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch); CID_OFFSET_MATCH(flags, flags); CID_OFFSET_MATCH(name, name); CID_OFFSET_MATCH(timeout_ms, timeout_ms); CID_OFFSET_MATCH(exit_dump_len, exit_dump_len); CID_OFFSET_MATCH(hotplug_seq, hotplug_seq); CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id); /* shared callbacks: the union view requires byte-for-byte offset match */ CID_OFFSET_MATCH(enqueue, enqueue); CID_OFFSET_MATCH(dequeue, dequeue); CID_OFFSET_MATCH(dispatch, dispatch); CID_OFFSET_MATCH(tick, tick); CID_OFFSET_MATCH(runnable, runnable); CID_OFFSET_MATCH(running, running); CID_OFFSET_MATCH(stopping, stopping); CID_OFFSET_MATCH(quiescent, quiescent); CID_OFFSET_MATCH(yield, yield); CID_OFFSET_MATCH(core_sched_before, core_sched_before); CID_OFFSET_MATCH(set_weight, set_weight); CID_OFFSET_MATCH(update_idle, update_idle); CID_OFFSET_MATCH(init_task, init_task); CID_OFFSET_MATCH(exit_task, exit_task); CID_OFFSET_MATCH(enable, enable); CID_OFFSET_MATCH(disable, disable); CID_OFFSET_MATCH(dump, dump); CID_OFFSET_MATCH(dump_task, dump_task); CID_OFFSET_MATCH(sub_attach, sub_attach); CID_OFFSET_MATCH(sub_detach, sub_detach); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit);#ifdef CONFIG_EXT_GROUP_SCHED CID_OFFSET_MATCH(cgroup_init, cgroup_init); CID_OFFSET_MATCH(cgroup_exit, cgroup_exit); CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move); CID_OFFSET_MATCH(cgroup_move, cgroup_move); CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move); CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight); CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth); CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle);#endif /* renamed callbacks must occupy the same slot as their cpu-form sibling */ CID_OFFSET_MATCH(select_cpu, select_cid); CID_OFFSET_MATCH(set_cpumask, set_cmask); CID_OFFSET_MATCH(cpu_online, cid_online); CID_OFFSET_MATCH(cpu_offline, cid_offline); CID_OFFSET_MATCH(dump_cpu, dump_cid); /* @priv tail must align since both share the same data block */ CID_OFFSET_MATCH(priv, priv); /* * cid-form must end exactly at @priv - validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields * past the BPF allocation would be UB. */ BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) != offsetofend(struct sched_ext_ops, priv));#undef CID_OFFSET_MATCH
/* * kfunc registration can't be done from init_sched_ext_class() as * register_btf_kfunc_id_set() needs most of the system to be up. * * Some kfuncs are context-sensitive and can only be called from * specific SCX ops. They are grouped into per-context BTF sets, each * registered with scx_kfunc_context_filter as its .filter callback. The * BPF core dedups identical filter pointers per hook * (btf_populate_kfunc_set()), so the filter is invoked exactly once per * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op * restrictions at verify time. */ if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_enqueue_dispatch)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_dispatch)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cpu_release)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_unlocked)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_unlocked)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_any)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_any)) || (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_any))) { pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); return ret; }
ret = scx_idle_init(); if (ret) { pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret); return ret; }
ret = scx_cid_kfunc_init(); if (ret) { pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret); return ret; }
ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); if (ret) { pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); return ret; }
ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid); if (ret) { pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret); return ret; }
ret = register_pm_notifier(&scx_pm_notifier); if (ret) { pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); return ret; }
scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); if (!scx_kset) { pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); return -ENOMEM; }
ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); if (ret < 0) { pr_err("sched_ext: Failed to add global attributes\n"); return ret; }
return 0;}__initcall(scx_init);