Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207// SPDX-License-Identifier: GPL-2.0-only OR MIT/* * Copyright (c) 2022 Red Hat. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * Authors: * Danilo Krummrich <dakr@redhat.com> * */
#include <drm/drm_drv.h>#include <drm/drm_gpuvm.h>#include <drm/drm_print.h>
#include <linux/export.h>#include <linux/interval_tree_generic.h>#include <linux/mm.h>
/** * DOC: Overview * * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a * GPU's virtual address (VA) space and manages the corresponding virtual * mappings represented by &drm_gpuva objects. It also keeps track of the * mapping's backing &drm_gem_object buffers. * * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing * all existing GPU VA mappings using this &drm_gem_object as backing buffer. * * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'. * * The GPU VA manager internally uses a rb-tree to manage the * &drm_gpuva mappings within a GPU's virtual address space. * * The &drm_gpuvm structure contains a special &drm_gpuva representing the * portion of VA space reserved by the kernel. This node is initialized together * with the GPU VA manager instance and removed when the GPU VA manager is * destroyed. * * In a typical application drivers would embed struct drm_gpuvm and * struct drm_gpuva within their own driver specific structures, there won't be * any memory allocations of its own nor memory allocations of &drm_gpuva * entries. * * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva * entries from within dma-fence signalling critical sections it is enough to * pre-allocate the &drm_gpuva structures. * * &drm_gem_objects which are private to a single VM can share a common * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec). * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in * the following called 'resv object', which serves as the container of the * GPUVM's shared &dma_resv. This resv object can be a driver specific * &drm_gem_object, such as the &drm_gem_object containing the root page table, * but it can also be a 'dummy' object, which can be allocated with * drm_gpuvm_resv_object_alloc(). * * In order to connect a struct drm_gpuva to its backing &drm_gem_object each * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each * &drm_gpuvm_bo contains a list of &drm_gpuva structures. * * A &drm_gpuvm_bo is an abstraction that represents a combination of a * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique. * This is ensured by the API through drm_gpuvm_bo_obtain() and * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this * particular combination. If not present, a new instance is created and linked * to the &drm_gem_object. * * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used * as entry for the &drm_gpuvm's lists of external and evicted objects. Those * lists are maintained in order to accelerate locking of dma-resv locks and * validation of evicted objects bound in a &drm_gpuvm. For instance, all * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in * order to validate all evicted &drm_gem_objects. It is also possible to lock * additional &drm_gem_objects by providing the corresponding parameters to * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making * use of helper functions such as drm_gpuvm_prepare_range() or * drm_gpuvm_prepare_objects(). * * Every bound &drm_gem_object is treated as external object when its &dma_resv * structure is different than the &drm_gpuvm's common &dma_resv structure. */
/** * DOC: Split and Merge * * Besides its capability to manage and represent a GPU VA space, the * GPU VA manager also provides functions to let the &drm_gpuvm calculate a * sequence of operations to satisfy a given map or unmap request. * * Therefore the DRM GPU VA manager provides an algorithm implementing splitting * and merging of existing GPU VA mappings with the ones that are requested to * be mapped or unmapped. This feature is required by the Vulkan API to * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this * as VM BIND. * * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks * containing map, unmap and remap operations for a given newly requested * mapping. The sequence of callbacks represents the set of operations to * execute in order to integrate the new mapping cleanly into the current state * of the GPU VA space. * * Depending on how the new GPU VA mapping intersects with the existing mappings * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount * of unmap operations, a maximum of two remap operations and a single map * operation. The caller might receive no callback at all if no operation is * required, e.g. if the requested mapping already exists in the exact same way. * * The single map operation represents the original map operation requested by * the caller. * * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the * &drm_gpuva to unmap is physically contiguous with the original mapping * request. Optionally, if 'keep' is set, drivers may keep the actual page table * entries for this &drm_gpuva, adding the missing page table entries only and * update the &drm_gpuvm's view of things accordingly. * * Drivers may do the same optimization, namely delta page table updates, also * for remap operations. This is possible since &drm_gpuva_op_remap consists of * one unmap operation and one or two map operations, such that drivers can * derive the page table update delta accordingly. * * Note that there can't be more than two existing mappings to split up, one at * the beginning and one at the end of the new mapping, hence there is a * maximum of two remap operations. * * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to * call back into the driver in order to unmap a range of GPU VA space. The * logic behind this function is way simpler though: For all existing mappings * enclosed by the given range unmap operations are created. For mappings which * are only partially located within the given range, remap operations are * created such that those mappings are split up and re-mapped partially. * * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(), * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used * to directly obtain an instance of struct drm_gpuva_ops containing a list of * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list * contains the &drm_gpuva_ops analogous to the callbacks one would receive when * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory * allocations are possible (e.g. to allocate GPU page tables) and once in the * dma-fence signalling critical path. * * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and * drm_gpuva_remove() may be used. These functions can safely be used from * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and * drm_gpuva_unmap() instead. * * The following diagram depicts the basic relationships of existing GPU VA * mappings, a newly requested mapping and the resulting mappings as implemented * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these. * * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs * could be kept. * * :: * * 0 a 1 * old: |-----------| (bo_offset=n) * * 0 a 1 * req: |-----------| (bo_offset=n) * * 0 a 1 * new: |-----------| (bo_offset=n) * * * 2) Requested mapping is identical, except for the BO offset, hence replace * the mapping. * * :: * * 0 a 1 * old: |-----------| (bo_offset=n) * * 0 a 1 * req: |-----------| (bo_offset=m) * * 0 a 1 * new: |-----------| (bo_offset=m) * * * 3) Requested mapping is identical, except for the backing BO, hence replace * the mapping. * * :: * * 0 a 1 * old: |-----------| (bo_offset=n) * * 0 b 1 * req: |-----------| (bo_offset=n) * * 0 b 1 * new: |-----------| (bo_offset=n) * * * 4) Existent mapping is a left aligned subset of the requested one, hence * replace the existing one. * * :: * * 0 a 1 * old: |-----| (bo_offset=n) * * 0 a 2 * req: |-----------| (bo_offset=n) * * 0 a 2 * new: |-----------| (bo_offset=n) * * .. note:: * We expect to see the same result for a request with a different BO * and/or non-contiguous BO offset. * * * 5) Requested mapping's range is a left aligned subset of the existing one, * but backed by a different BO. Hence, map the requested mapping and split * the existing one adjusting its BO offset. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 0 b 1 * req: |-----| (bo_offset=n) * * 0 b 1 a' 2 * new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1) * * .. note:: * We expect to see the same result for a request with a different BO * and/or non-contiguous BO offset. * * * 6) Existent mapping is a superset of the requested mapping. Split it up, but * indicate that the backing PTEs could be kept. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 0 a 1 * req: |-----| (bo_offset=n) * * 0 a 1 a' 2 * new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1) * * * 7) Requested mapping's range is a right aligned subset of the existing one, * but backed by a different BO. Hence, map the requested mapping and split * the existing one, without adjusting the BO offset. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 b 2 * req: |-----| (bo_offset=m) * * 0 a 1 b 2 * new: |-----|-----| (a.bo_offset=n,b.bo_offset=m) * * * 8) Existent mapping is a superset of the requested mapping. Split it up, but * indicate that the backing PTEs could be kept. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 a 2 * req: |-----| (bo_offset=n+1) * * 0 a' 1 a 2 * new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1) * * * 9) Existent mapping is overlapped at the end by the requested mapping backed * by a different BO. Hence, map the requested mapping and split up the * existing one, without adjusting the BO offset. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 b 3 * req: |-----------| (bo_offset=m) * * 0 a 1 b 3 * new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m) * * * 10) Existent mapping is overlapped by the requested mapping, both having the * same backing BO with a contiguous offset. Indicate the backing PTEs of * the old mapping could be kept. * * :: * * 0 a 2 * old: |-----------| (bo_offset=n) * * 1 a 3 * req: |-----------| (bo_offset=n+1) * * 0 a' 1 a 3 * new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1) * * * 11) Requested mapping's range is a centered subset of the existing one * having a different backing BO. Hence, map the requested mapping and split * up the existing one in two mappings, adjusting the BO offset of the right * one accordingly. * * :: * * 0 a 3 * old: |-----------------| (bo_offset=n) * * 1 b 2 * req: |-----| (bo_offset=m) * * 0 a 1 b 2 a' 3 * new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2) * * * 12) Requested mapping is a contiguous subset of the existing one. Split it * up, but indicate that the backing PTEs could be kept. * * :: * * 0 a 3 * old: |-----------------| (bo_offset=n) * * 1 a 2 * req: |-----| (bo_offset=n+1) * * 0 a' 1 a 2 a'' 3 * old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2) * * * 13) Existent mapping is a right aligned subset of the requested one, hence * replace the existing one. * * :: * * 1 a 2 * old: |-----| (bo_offset=n+1) * * 0 a 2 * req: |-----------| (bo_offset=n) * * 0 a 2 * new: |-----------| (bo_offset=n) * * .. note:: * We expect to see the same result for a request with a different bo * and/or non-contiguous bo_offset. * * * 14) Existent mapping is a centered subset of the requested one, hence * replace the existing one. * * :: * * 1 a 2 * old: |-----| (bo_offset=n+1) * * 0 a 3 * req: |----------------| (bo_offset=n) * * 0 a 3 * new: |----------------| (bo_offset=n) * * .. note:: * We expect to see the same result for a request with a different bo * and/or non-contiguous bo_offset. * * * 15) Existent mappings is overlapped at the beginning by the requested mapping * backed by a different BO. Hence, map the requested mapping and split up * the existing one, adjusting its BO offset accordingly. * * :: * * 1 a 3 * old: |-----------| (bo_offset=n) * * 0 b 2 * req: |-----------| (bo_offset=m) * * 0 b 2 a' 3 * new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2) */
/** * DOC: Madvise Logic - Splitting and Traversal * * This logic handles GPU VA range updates by generating remap and map operations * without performing unmaps or merging existing mappings. * * 1) The requested range lies entirely within a single drm_gpuva. The logic splits * the existing mapping at the start and end boundaries and inserts a new map. * * :: * a start end b * pre: |-----------------------| * drm_gpuva1 * * a start end b * new: |-----|=========|-------| * remap map remap * * one REMAP and one MAP : Same behaviour as SPLIT and MERGE * * 2) The requested range spans multiple drm_gpuva regions. The logic traverses * across boundaries, remapping the start and end segments, and inserting two * map operations to cover the full range. * * :: a start b c end d * pre: |------------------|--------------|------------------| * drm_gpuva1 drm_gpuva2 drm_gpuva3 * * a start b c end d * new: |-------|==========|--------------|========|---------| * remap1 map1 drm_gpuva2 map2 remap2 * * two REMAPS and two MAPS * * 3) Either start or end lies within a drm_gpuva. A single remap and map operation * are generated to update the affected portion. * * * :: a/start b c end d * pre: |------------------|--------------|------------------| * drm_gpuva1 drm_gpuva2 drm_gpuva3 * * a/start b c end d * new: |------------------|--------------|========|---------| * drm_gpuva1 drm_gpuva2 map1 remap1 * * :: a start b c/end d * pre: |------------------|--------------|------------------| * drm_gpuva1 drm_gpuva2 drm_gpuva3 * * a start b c/end d * new: |-------|==========|--------------|------------------| * remap1 map1 drm_gpuva2 drm_gpuva3 * * one REMAP and one MAP * * 4) Both start and end align with existing drm_gpuva boundaries. No operations * are needed as the range is already covered. * * 5) No existing drm_gpuvas. No operations. * * Unlike drm_gpuvm_sm_map_ops_create, this logic avoids unmaps and merging, * focusing solely on remap and map operations for efficient traversal and update. */
/** * DOC: Locking * * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of * locking itself, it is the drivers responsibility to take care about locking. * Drivers might want to protect the following operations: inserting, removing * and iterating &drm_gpuva objects as well as generating all kinds of * operations, such as split / merge or prefetch. * * DRM GPUVM also does not take care of the locking of the backing * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by * itself; drivers are responsible to enforce mutual exclusion using either the * GEMs dma_resv lock or the GEMs gpuva.lock mutex. * * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put(). * * The latter is required since on creation and destruction of a &drm_gpuvm_bo * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list. * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and * &drm_gem_object must be able to observe previous creations and destructions * of &drm_gpuvm_bos in order to keep instances unique. * * The &drm_gpuvm's lists for keeping track of external and evicted objects are * protected against concurrent insertion / removal and iteration internally. * * However, drivers still need ensure to protect concurrent calls to functions * iterating those lists, namely drm_gpuvm_prepare_objects() and * drm_gpuvm_validate(). * * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate * that the corresponding &dma_resv locks are held in order to protect the * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and * the corresponding lockdep checks are enabled. This is an optimization for * drivers which are capable of taking the corresponding &dma_resv locks and * hence do not require internal locking. */
/** * DOC: Examples * * This section gives two examples on how to let the DRM GPUVA Manager generate * &drm_gpuva_op in order to satisfy a given map or unmap request and how to * make use of them. * * The below code is strictly limited to illustrate the generic usage pattern. * To maintain simplicity, it doesn't make use of any abstractions for common * code, different (asynchronous) stages with fence signalling critical paths, * any other helpers or error handling in terms of freeing memory and dropping * previously taken locks. * * 1) Obtain a list of &drm_gpuva_op to create a new mapping:: * * // Allocates a new &drm_gpuva. * struct drm_gpuva * driver_gpuva_alloc(void); * * // Typically drivers would embed the &drm_gpuvm and &drm_gpuva * // structure in individual driver structures and lock the dma-resv with * // drm_exec or similar helpers. * int driver_mapping_create(struct drm_gpuvm *gpuvm, * u64 addr, u64 range, * struct drm_gem_object *obj, u64 offset) * { * struct drm_gpuvm_map_req map_req = { * .map.va.addr = addr, * .map.va.range = range, * .map.gem.obj = obj, * .map.gem.offset = offset, * }; * struct drm_gpuva_ops *ops; * struct drm_gpuva_op *op * struct drm_gpuvm_bo *vm_bo; * * driver_lock_va_space(); * ops = drm_gpuvm_sm_map_ops_create(gpuvm, &map_req); * if (IS_ERR(ops)) * return PTR_ERR(ops); * * vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj); * if (IS_ERR(vm_bo)) * return PTR_ERR(vm_bo); * * drm_gpuva_for_each_op(op, ops) { * struct drm_gpuva *va; * * switch (op->op) { * case DRM_GPUVA_OP_MAP: * va = driver_gpuva_alloc(); * if (!va) * ; // unwind previous VA space updates, * // free memory and unlock * * driver_vm_map(); * drm_gpuva_map(gpuvm, va, &op->map); * drm_gpuva_link(va, vm_bo); * * break; * case DRM_GPUVA_OP_REMAP: { * struct drm_gpuva *prev = NULL, *next = NULL; * * va = op->remap.unmap->va; * * if (op->remap.prev) { * prev = driver_gpuva_alloc(); * if (!prev) * ; // unwind previous VA space * // updates, free memory and * // unlock * } * * if (op->remap.next) { * next = driver_gpuva_alloc(); * if (!next) * ; // unwind previous VA space * // updates, free memory and * // unlock * } * * driver_vm_remap(); * drm_gpuva_remap(prev, next, &op->remap); * * if (prev) * drm_gpuva_link(prev, va->vm_bo); * if (next) * drm_gpuva_link(next, va->vm_bo); * drm_gpuva_unlink(va); * * break; * } * case DRM_GPUVA_OP_UNMAP: * va = op->unmap->va; * * driver_vm_unmap(); * drm_gpuva_unlink(va); * drm_gpuva_unmap(&op->unmap); * * break; * default: * break; * } * } * drm_gpuvm_bo_put(vm_bo); * driver_unlock_va_space(); * * return 0; * } * * 2) Receive a callback for each &drm_gpuva_op to create a new mapping:: * * struct driver_context { * struct drm_gpuvm *gpuvm; * struct drm_gpuvm_bo *vm_bo; * struct drm_gpuva *new_va; * struct drm_gpuva *prev_va; * struct drm_gpuva *next_va; * }; * * // ops to pass to drm_gpuvm_init() * static const struct drm_gpuvm_ops driver_gpuvm_ops = { * .sm_step_map = driver_gpuva_map, * .sm_step_remap = driver_gpuva_remap, * .sm_step_unmap = driver_gpuva_unmap, * }; * * // Typically drivers would embed the &drm_gpuvm and &drm_gpuva * // structure in individual driver structures and lock the dma-resv with * // drm_exec or similar helpers. * int driver_mapping_create(struct drm_gpuvm *gpuvm, * u64 addr, u64 range, * struct drm_gem_object *obj, u64 offset) * { * struct driver_context ctx; * struct drm_gpuvm_bo *vm_bo; * struct drm_gpuva_ops *ops; * struct drm_gpuva_op *op; * int ret = 0; * * ctx.gpuvm = gpuvm; * * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL); * ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj); * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) { * ret = -ENOMEM; * goto out; * } * * // Typically protected with a driver specific GEM gpuva lock * // used in the fence signaling path for drm_gpuva_link() and * // drm_gpuva_unlink(), hence pre-allocate. * ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo); * * driver_lock_va_space(); * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset); * driver_unlock_va_space(); * * out: * drm_gpuvm_bo_put(ctx.vm_bo); * kfree(ctx.new_va); * kfree(ctx.prev_va); * kfree(ctx.next_va); * return ret; * } * * int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx) * { * struct driver_context *ctx = __ctx; * * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map); * * drm_gpuva_link(ctx->new_va, ctx->vm_bo); * * // prevent the new GPUVA from being freed in * // driver_mapping_create() * ctx->new_va = NULL; * * return 0; * } * * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx) * { * struct driver_context *ctx = __ctx; * struct drm_gpuva *va = op->remap.unmap->va; * * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap); * * if (op->remap.prev) { * drm_gpuva_link(ctx->prev_va, va->vm_bo); * ctx->prev_va = NULL; * } * * if (op->remap.next) { * drm_gpuva_link(ctx->next_va, va->vm_bo); * ctx->next_va = NULL; * } * * drm_gpuva_unlink(va); * kfree(va); * * return 0; * } * * int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx) * { * drm_gpuva_unlink(op->unmap.va); * drm_gpuva_unmap(&op->unmap); * kfree(op->unmap.va); * * return 0; * } */
/** * get_next_vm_bo_from_list() - get the next vm_bo element * @__gpuvm: the &drm_gpuvm * @__list_name: the name of the list we're iterating on * @__local_list: a pointer to the local list used to store already iterated items * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list() * * This helper is here to provide lockless list iteration. Lockless as in, the * iterator releases the lock immediately after picking the first element from * the list, so list insertion and deletion can happen concurrently. * * Elements popped from the original list are kept in a local list, so removal * and is_empty checks can still happen while we're iterating the list. */#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \ ({ \ struct drm_gpuvm_bo *__vm_bo = NULL; \ \ drm_gpuvm_bo_put(__prev_vm_bo); \ \ spin_lock(&(__gpuvm)->__list_name.lock); \ if (!(__gpuvm)->__list_name.local_list) \ (__gpuvm)->__list_name.local_list = __local_list; \ else \ drm_WARN_ON((__gpuvm)->drm, \ (__gpuvm)->__list_name.local_list != __local_list); \ \ while (!list_empty(&(__gpuvm)->__list_name.list)) { \ __vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \ struct drm_gpuvm_bo, \ list.entry.__list_name); \ if (kref_get_unless_zero(&__vm_bo->kref)) { \ list_move_tail(&(__vm_bo)->list.entry.__list_name, \ __local_list); \ break; \ } else { \ list_del_init(&(__vm_bo)->list.entry.__list_name); \ __vm_bo = NULL; \ } \ } \ spin_unlock(&(__gpuvm)->__list_name.lock); \ \ __vm_bo; \ })
/** * for_each_vm_bo_in_list() - internal vm_bo list iterator * @__gpuvm: the &drm_gpuvm * @__list_name: the name of the list we're iterating on * @__local_list: a pointer to the local list used to store already iterated items * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step * * This helper is here to provide lockless list iteration. Lockless as in, the * iterator releases the lock immediately after picking the first element from the * list, hence list insertion and deletion can happen concurrently. * * It is not allowed to re-assign the vm_bo pointer from inside this loop. * * Typical use: * * struct drm_gpuvm_bo *vm_bo; * LIST_HEAD(my_local_list); * * ret = 0; * for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) { * ret = do_something_with_vm_bo(..., vm_bo); * if (ret) * break; * } * // Drop ref in case we break out of the loop. * drm_gpuvm_bo_put(vm_bo); * restore_vm_bo_list(gpuvm, <list_name>, &my_local_list); * * * Only used for internal list iterations, not meant to be exposed to the outside * world. */#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \ for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \ __local_list, NULL); \ __vm_bo; \ __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \ __local_list, __vm_bo))
static void__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock, struct list_head *list, struct list_head **local_list){ /* Merge back the two lists, moving local list elements to the * head to preserve previous ordering, in case it matters. */ spin_lock(lock); if (*local_list) { list_splice(*local_list, list); *local_list = NULL; } spin_unlock(lock);}
/** * restore_vm_bo_list() - move vm_bo elements back to their original list * @__gpuvm: the &drm_gpuvm * @__list_name: the name of the list we're iterating on * * When we're done iterating a vm_bo list, we should call restore_vm_bo_list() * to restore the original state and let new iterations take place. */#define restore_vm_bo_list(__gpuvm, __list_name) \ __restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \ &(__gpuvm)->__list_name.list, \ &(__gpuvm)->__list_name.local_list)
static voidcond_spin_lock(spinlock_t *lock, bool cond){ if (cond) spin_lock(lock);}
static voidcond_spin_unlock(spinlock_t *lock, bool cond){ if (cond) spin_unlock(lock);}
static void__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock, struct list_head *entry, struct list_head *list){ cond_spin_lock(lock, !!lock); if (list_empty(entry)) list_add_tail(entry, list); cond_spin_unlock(lock, !!lock);}
/** * drm_gpuvm_bo_is_zombie() - check whether this vm_bo is scheduled for cleanup * @vm_bo: the &drm_gpuvm_bo * * When a vm_bo is scheduled for cleanup using the bo_defer list, it is not * immediately removed from the evict and extobj lists. Therefore, anyone * iterating these lists should skip entries that are being destroyed. * * Checking the refcount without incrementing it is okay as long as the lock * protecting the evict/extobj list is held for as long as you are using the * vm_bo, because even if the refcount hits zero while you are using it, freeing * the vm_bo requires taking the list's lock. * * Zombie entries can be observed on the evict and extobj lists regardless of * whether DRM_GPUVM_RESV_PROTECTED is used, but they remain on the lists for a * longer time when the resv lock is used because we can't take the resv lock * during run_job() in immediate mode, meaning that they need to remain on the * lists until drm_gpuvm_bo_deferred_cleanup() is called. */static booldrm_gpuvm_bo_is_zombie(struct drm_gpuvm_bo *vm_bo){ return !kref_read(&vm_bo->kref);}
/** * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list * @__vm_bo: the &drm_gpuvm_bo * @__list_name: the name of the list to insert into * @__lock: whether to lock with the internal spinlock * * Inserts the given @__vm_bo into the list specified by @__list_name. */#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \ __drm_gpuvm_bo_list_add((__vm_bo)->vm, \ __lock ? &(__vm_bo)->vm->__list_name.lock : \ NULL, \ &(__vm_bo)->list.entry.__list_name, \ &(__vm_bo)->vm->__list_name.list)
static void__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock, struct list_head *entry, bool init){ cond_spin_lock(lock, !!lock); if (init) { if (!list_empty(entry)) list_del_init(entry); } else { list_del(entry); } cond_spin_unlock(lock, !!lock);}
/** * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list * @__vm_bo: the &drm_gpuvm_bo * @__list_name: the name of the list to insert into * @__lock: whether to lock with the internal spinlock * * Removes the given @__vm_bo from the list specified by @__list_name. */#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \ __drm_gpuvm_bo_list_del((__vm_bo)->vm, \ __lock ? &(__vm_bo)->vm->__list_name.lock : \ NULL, \ &(__vm_bo)->list.entry.__list_name, \ true)
/** * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list * @__vm_bo: the &drm_gpuvm_bo * @__list_name: the name of the list to insert into * @__lock: whether to lock with the internal spinlock * * Removes the given @__vm_bo from the list specified by @__list_name. */#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \ __drm_gpuvm_bo_list_del((__vm_bo)->vm, \ __lock ? &(__vm_bo)->vm->__list_name.lock : \ NULL, \ &(__vm_bo)->list.entry.__list_name, \ false)
#define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node)
#define GPUVA_START(node) ((node)->va.addr)#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
/* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain * about this. */INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last, GPUVA_START, GPUVA_LAST, static __maybe_unused, drm_gpuva_it)
static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va);static void __drm_gpuva_remove(struct drm_gpuva *va);
static booldrm_gpuvm_check_overflow(u64 addr, u64 range){ u64 end;
return check_add_overflow(addr, range, &end);}
static booldrm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range), "GPUVA address limited to %zu bytes.\n", sizeof(addr));}
static booldrm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ u64 end = addr + range; u64 mm_start = gpuvm->mm_start; u64 mm_end = mm_start + gpuvm->mm_range;
return addr >= mm_start && end <= mm_end;}
static booldrm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ u64 end = addr + range; u64 kstart = gpuvm->kernel_alloc_node.va.addr; u64 krange = gpuvm->kernel_alloc_node.va.range; u64 kend = kstart + krange;
return krange && addr < kend && kstart < end;}
/** * drm_gpuvm_range_valid() - checks whether the given range is valid for the * given &drm_gpuvm * @gpuvm: the GPUVM to check the range for * @addr: the base address * @range: the range starting from the base address * * Checks whether the range is within the GPUVM's managed boundaries. * * Returns: true for a valid range, false otherwise */booldrm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ return !drm_gpuvm_check_overflow(addr, range) && drm_gpuvm_in_mm_range(gpuvm, addr, range) && !drm_gpuvm_in_kernel_node(gpuvm, addr, range);}EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid);
static voiddrm_gpuvm_gem_object_free(struct drm_gem_object *obj){ drm_gem_object_release(obj); kfree(obj);}
static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = { .free = drm_gpuvm_gem_object_free,};
/** * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object * @drm: the drivers &drm_device * * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in * order to serve as root GEM object providing the &drm_resv shared across * &drm_gem_objects local to a single GPUVM. * * Returns: the &drm_gem_object on success, NULL on failure */struct drm_gem_object *drm_gpuvm_resv_object_alloc(struct drm_device *drm){ struct drm_gem_object *obj;
obj = kzalloc_obj(*obj); if (!obj) return NULL;
obj->funcs = &drm_gpuvm_object_funcs; drm_gem_private_object_init(drm, obj, 0);
return obj;}EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
/** * drm_gpuvm_init() - initialize a &drm_gpuvm * @gpuvm: pointer to the &drm_gpuvm to initialize * @name: the name of the GPU VA space * @flags: the &drm_gpuvm_flags for this GPUVM * @drm: the &drm_device this VM resides in * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv * @start_offset: the start offset of the GPU VA space * @range: the size of the GPU VA space * @reserve_offset: the start of the kernel reserved GPU VA area * @reserve_range: the size of the kernel reserved GPU VA area * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap * * The &drm_gpuvm must be initialized with this function before use. * * Note that @gpuvm must be cleared to 0 before calling this function. The given * &name is expected to be managed by the surrounding driver structures. */voiddrm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, enum drm_gpuvm_flags flags, struct drm_device *drm, struct drm_gem_object *r_obj, u64 start_offset, u64 range, u64 reserve_offset, u64 reserve_range, const struct drm_gpuvm_ops *ops){ gpuvm->rb.tree = RB_ROOT_CACHED; INIT_LIST_HEAD(&gpuvm->rb.list);
INIT_LIST_HEAD(&gpuvm->extobj.list); spin_lock_init(&gpuvm->extobj.lock);
INIT_LIST_HEAD(&gpuvm->evict.list); spin_lock_init(&gpuvm->evict.lock);
init_llist_head(&gpuvm->bo_defer);
kref_init(&gpuvm->kref);
gpuvm->name = name ? name : "unknown"; gpuvm->flags = flags; gpuvm->ops = ops; gpuvm->drm = drm; gpuvm->r_obj = r_obj;
drm_dev_get(drm); drm_gem_object_get(r_obj);
drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); gpuvm->mm_start = start_offset; gpuvm->mm_range = range;
memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); if (reserve_range) { gpuvm->kernel_alloc_node.va.addr = reserve_offset; gpuvm->kernel_alloc_node.va.range = reserve_range;
if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset, reserve_range))) __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node); }}EXPORT_SYMBOL_GPL(drm_gpuvm_init);
static voiddrm_gpuvm_fini(struct drm_gpuvm *gpuvm){ gpuvm->name = NULL;
if (gpuvm->kernel_alloc_node.va.range) __drm_gpuva_remove(&gpuvm->kernel_alloc_node);
drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), "GPUVA tree is not empty, potentially leaking memory.\n");
drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list), "Extobj list should be empty.\n"); drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list), "Evict list should be empty.\n"); drm_WARN(gpuvm->drm, !llist_empty(&gpuvm->bo_defer), "VM BO cleanup list should be empty.\n");
drm_gem_object_put(gpuvm->r_obj);}
static voiddrm_gpuvm_free(struct kref *kref){ struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref); struct drm_device *drm = gpuvm->drm;
drm_gpuvm_fini(gpuvm);
if (drm_WARN_ON(drm, !gpuvm->ops->vm_free)) return;
gpuvm->ops->vm_free(gpuvm); drm_dev_put(drm);}
/** * drm_gpuvm_put() - drop a struct drm_gpuvm reference * @gpuvm: the &drm_gpuvm to release the reference of * * This releases a reference to @gpuvm. * * This function may be called from atomic context. */voiddrm_gpuvm_put(struct drm_gpuvm *gpuvm){ if (gpuvm) kref_put(&gpuvm->kref, drm_gpuvm_free);}EXPORT_SYMBOL_GPL(drm_gpuvm_put);
static intexec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, unsigned int num_fences){ return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) : drm_exec_lock_obj(exec, obj);}
/** * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv * @gpuvm: the &drm_gpuvm * @exec: the &drm_exec context * @num_fences: the amount of &dma_fences to reserve * * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if * @num_fences is zero drm_exec_lock_obj() is called instead. * * Using this function directly, it is the drivers responsibility to call * drm_exec_init() and drm_exec_fini() accordingly. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences){ return exec_prepare_obj(exec, gpuvm->r_obj, num_fences);}EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm);
static int__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences){ struct drm_gpuvm_bo *vm_bo; LIST_HEAD(extobjs); int ret = 0;
for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) { ret = exec_prepare_obj(exec, vm_bo->obj, num_fences); if (ret) break; } /* Drop ref in case we break out of the loop. */ drm_gpuvm_bo_put(vm_bo); restore_vm_bo_list(gpuvm, extobj);
return ret;}
static intdrm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences){ struct drm_gpuvm_bo *vm_bo; int ret = 0;
drm_gpuvm_resv_assert_held(gpuvm); list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) { if (drm_gpuvm_bo_is_zombie(vm_bo)) continue;
ret = exec_prepare_obj(exec, vm_bo->obj, num_fences); if (ret) break;
if (vm_bo->evicted) drm_gpuvm_bo_list_add(vm_bo, evict, false); }
return ret;}
/** * drm_gpuvm_prepare_objects() - prepare all associated BOs * @gpuvm: the &drm_gpuvm * @exec: the &drm_exec locking context * @num_fences: the amount of &dma_fences to reserve * * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given * &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj() * is called instead. * * Using this function directly, it is the drivers responsibility to call * drm_exec_init() and drm_exec_fini() accordingly. * * Note: This function is safe against concurrent insertion and removal of * external objects, however it is not safe against concurrent usage itself. * * Drivers need to make sure to protect this case with either an outer VM lock * or by calling drm_gpuvm_prepare_vm() before this function within the * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures * mutual exclusion. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences){ if (drm_gpuvm_resv_protected(gpuvm)) return drm_gpuvm_prepare_objects_locked(gpuvm, exec, num_fences); else return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);}EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);
/** * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range * @gpuvm: the &drm_gpuvm * @exec: the &drm_exec locking context * @addr: the start address within the VA space * @range: the range to iterate within the VA space * @num_fences: the amount of &dma_fences to reserve * * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr * and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called * instead. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, u64 addr, u64 range, unsigned int num_fences){ struct drm_gpuva *va; u64 end = addr + range; int ret;
drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { struct drm_gem_object *obj = va->gem.obj;
if (unlikely(!obj)) continue;
ret = exec_prepare_obj(exec, obj, num_fences); if (ret) return ret; }
return 0;}EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);
/** * drm_gpuvm_exec_lock() - lock all dma-resv of all associated BOs * @vm_exec: the &drm_gpuvm_exec wrapper * * Acquires all dma-resv locks of all &drm_gem_objects the given * &drm_gpuvm contains mappings of. * * Additionally, when calling this function with struct drm_gpuvm_exec::extra * being set the driver receives the given @fn callback to lock additional * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers * would call drm_exec_prepare_obj() from within this callback. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec){ struct drm_gpuvm *gpuvm = vm_exec->vm; struct drm_exec *exec = &vm_exec->exec; unsigned int num_fences = vm_exec->num_fences; int ret;
drm_exec_init(exec, vm_exec->flags, 0);
drm_exec_until_all_locked(exec) { ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences); drm_exec_retry_on_contention(exec); if (ret) goto err;
ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences); drm_exec_retry_on_contention(exec); if (ret) goto err;
if (vm_exec->extra.fn) { ret = vm_exec->extra.fn(vm_exec); drm_exec_retry_on_contention(exec); if (ret) goto err; } }
return 0;
err: drm_exec_fini(exec); return ret;}EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);
static intfn_lock_array(struct drm_gpuvm_exec *vm_exec){ struct { struct drm_gem_object **objs; unsigned int num_objs; } *args = vm_exec->extra.priv;
return drm_exec_prepare_array(&vm_exec->exec, args->objs, args->num_objs, vm_exec->num_fences);}
/** * drm_gpuvm_exec_lock_array() - lock all dma-resv of all associated BOs * @vm_exec: the &drm_gpuvm_exec wrapper * @objs: additional &drm_gem_objects to lock * @num_objs: the number of additional &drm_gem_objects to lock * * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm * contains mappings of, plus the ones given through @objs. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, struct drm_gem_object **objs, unsigned int num_objs){ struct { struct drm_gem_object **objs; unsigned int num_objs; } args;
args.objs = objs; args.num_objs = num_objs;
vm_exec->extra.fn = fn_lock_array; vm_exec->extra.priv = &args;
return drm_gpuvm_exec_lock(vm_exec);}EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);
/** * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range * @vm_exec: the &drm_gpuvm_exec wrapper * @addr: the start address within the VA space * @range: the range to iterate within the VA space * * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and * @addr + @range. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, u64 addr, u64 range){ struct drm_gpuvm *gpuvm = vm_exec->vm; struct drm_exec *exec = &vm_exec->exec; int ret;
drm_exec_init(exec, vm_exec->flags, 0);
drm_exec_until_all_locked(exec) { ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range, vm_exec->num_fences); drm_exec_retry_on_contention(exec); if (ret) goto err; }
return ret;
err: drm_exec_fini(exec); return ret;}EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);
static int__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec){ const struct drm_gpuvm_ops *ops = gpuvm->ops; struct drm_gpuvm_bo *vm_bo; LIST_HEAD(evict); int ret = 0;
for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) { ret = ops->vm_bo_validate(vm_bo, exec); if (ret) break; } /* Drop ref in case we break out of the loop. */ drm_gpuvm_bo_put(vm_bo); restore_vm_bo_list(gpuvm, evict);
return ret;}
static intdrm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec){ const struct drm_gpuvm_ops *ops = gpuvm->ops; struct drm_gpuvm_bo *vm_bo, *next; int ret = 0;
drm_gpuvm_resv_assert_held(gpuvm);
list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list, list.entry.evict) { if (drm_gpuvm_bo_is_zombie(vm_bo)) continue;
ret = ops->vm_bo_validate(vm_bo, exec); if (ret) break;
dma_resv_assert_held(vm_bo->obj->resv); if (!vm_bo->evicted) drm_gpuvm_bo_list_del_init(vm_bo, evict, false); }
return ret;}
/** * drm_gpuvm_validate() - validate all BOs marked as evicted * @gpuvm: the &drm_gpuvm to validate evicted BOs * @exec: the &drm_exec instance used for locking the GPUVM * * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer * objects being mapped in the given &drm_gpuvm. * * Returns: 0 on success, negative error code on failure. */intdrm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec){ const struct drm_gpuvm_ops *ops = gpuvm->ops;
if (unlikely(!ops || !ops->vm_bo_validate)) return -EOPNOTSUPP;
if (drm_gpuvm_resv_protected(gpuvm)) return drm_gpuvm_validate_locked(gpuvm, exec); else return __drm_gpuvm_validate(gpuvm, exec);}EXPORT_SYMBOL_GPL(drm_gpuvm_validate);
/** * drm_gpuvm_resv_add_fence - add fence to private and all extobj * dma-resv * @gpuvm: the &drm_gpuvm to add a fence to * @exec: the &drm_exec locking context * @fence: fence to add * @private_usage: private dma-resv usage * @extobj_usage: extobj dma-resv usage */voiddrm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, struct drm_exec *exec, struct dma_fence *fence, enum dma_resv_usage private_usage, enum dma_resv_usage extobj_usage){ struct drm_gem_object *obj;
drm_exec_for_each_locked_object(exec, obj) { dma_resv_assert_held(obj->resv); dma_resv_add_fence(obj->resv, fence, drm_gpuvm_is_extobj(gpuvm, obj) ? extobj_usage : private_usage); }}EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);
/** * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo * @gpuvm: The &drm_gpuvm the @obj is mapped in. * @obj: The &drm_gem_object being mapped in the @gpuvm. * * If provided by the driver, this function uses the &drm_gpuvm_ops * vm_bo_alloc() callback to allocate. * * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure */struct drm_gpuvm_bo *drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj){ const struct drm_gpuvm_ops *ops = gpuvm->ops; struct drm_gpuvm_bo *vm_bo;
if (ops && ops->vm_bo_alloc) vm_bo = ops->vm_bo_alloc(); else vm_bo = kzalloc_obj(*vm_bo);
if (unlikely(!vm_bo)) return NULL;
vm_bo->vm = drm_gpuvm_get(gpuvm); vm_bo->obj = obj; drm_gem_object_get(obj);
kref_init(&vm_bo->kref); INIT_LIST_HEAD(&vm_bo->list.gpuva); INIT_LIST_HEAD(&vm_bo->list.entry.gem);
INIT_LIST_HEAD(&vm_bo->list.entry.extobj); INIT_LIST_HEAD(&vm_bo->list.entry.evict); init_llist_node(&vm_bo->list.entry.bo_defer);
return vm_bo;}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
/* * drm_gpuvm_bo_destroy_not_in_lists() - final part of drm_gpuvm_bo cleanup * @vm_bo: the &drm_gpuvm_bo to destroy * * It is illegal to call this method if the @vm_bo is present in the GEMs gpuva * list, the extobj list, or the evicted list. * * Note that this puts a refcount on the GEM object, which may destroy the GEM * object if the refcount reaches zero. It's illegal for this to happen if the * caller holds the GEMs gpuva mutex because it would free the mutex. */static voiddrm_gpuvm_bo_destroy_not_in_lists(struct drm_gpuvm_bo *vm_bo){ struct drm_gpuvm *gpuvm = vm_bo->vm; const struct drm_gpuvm_ops *ops = gpuvm->ops; struct drm_gem_object *obj = vm_bo->obj;
if (ops && ops->vm_bo_free) ops->vm_bo_free(vm_bo); else kfree(vm_bo);
drm_gpuvm_put(gpuvm); drm_gem_object_put(obj);}
static voiddrm_gpuvm_bo_destroy_not_in_lists_kref(struct kref *kref){ struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo, kref);
drm_gpuvm_bo_destroy_not_in_lists(vm_bo);}
static voiddrm_gpuvm_bo_destroy(struct kref *kref){ struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo, kref); struct drm_gpuvm *gpuvm = vm_bo->vm; bool lock = !drm_gpuvm_resv_protected(gpuvm);
if (!lock) drm_gpuvm_resv_assert_held(gpuvm);
drm_gpuvm_bo_list_del(vm_bo, extobj, lock); drm_gpuvm_bo_list_del(vm_bo, evict, lock);
drm_gem_gpuva_assert_lock_held(gpuvm, vm_bo->obj); list_del(&vm_bo->list.entry.gem);
drm_gpuvm_bo_destroy_not_in_lists(vm_bo);}
/** * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference * @vm_bo: the &drm_gpuvm_bo to release the reference of * * This releases a reference to @vm_bo. * * If the reference count drops to zero, the &gpuvm_bo is destroyed, which * includes removing it from the GEMs gpuva list. Hence, if a call to this * function can potentially let the reference count drop to zero the caller must * hold the lock that the GEM uses for its gpuva list (either the GEM's * dma-resv or gpuva.lock mutex). * * This function may only be called from non-atomic context. * * Returns: true if vm_bo was destroyed, false otherwise. */booldrm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo){ might_sleep();
if (vm_bo) return !!kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy);
return false;}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);
/* * drm_gpuvm_bo_into_zombie() - called when the vm_bo becomes a zombie due to * deferred cleanup * * If deferred cleanup is used, then this must be called right after the vm_bo * refcount drops to zero. Must be called with GEM mutex held. After releasing * the GEM mutex, drm_gpuvm_bo_defer_zombie_cleanup() must be called. */static voiddrm_gpuvm_bo_into_zombie(struct kref *kref){ struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo, kref);
if (!drm_gpuvm_resv_protected(vm_bo->vm)) { drm_gpuvm_bo_list_del(vm_bo, extobj, true); drm_gpuvm_bo_list_del(vm_bo, evict, true); }
list_del(&vm_bo->list.entry.gem);}
/* * drm_gpuvm_bo_defer_zombie_cleanup() - adds a new zombie vm_bo to the * bo_defer list * * Called after drm_gpuvm_bo_into_zombie(). GEM mutex must not be held. * * It's important that the GEM stays alive for the duration in which we hold * the mutex, but the instant we add the vm_bo to bo_defer, another thread * might call drm_gpuvm_bo_deferred_cleanup() and put the GEM. Therefore, to * avoid kfreeing a mutex we are holding, the GEM mutex must be released * *before* calling this function. */static voiddrm_gpuvm_bo_defer_zombie_cleanup(struct drm_gpuvm_bo *vm_bo){ llist_add(&vm_bo->list.entry.bo_defer, &vm_bo->vm->bo_defer);}
static voiddrm_gpuvm_bo_defer_free(struct kref *kref){ struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo, kref);
drm_gpuvm_bo_into_zombie(kref); mutex_unlock(&vm_bo->obj->gpuva.lock); drm_gpuvm_bo_defer_zombie_cleanup(vm_bo);}
/** * drm_gpuvm_bo_put_deferred() - drop a struct drm_gpuvm_bo reference with * deferred cleanup * @vm_bo: the &drm_gpuvm_bo to release the reference of * * This releases a reference to @vm_bo. * * This might take and release the GEMs GPUVA lock. You should call * drm_gpuvm_bo_deferred_cleanup() later to complete the cleanup process. * * Returns: true if vm_bo is being destroyed, false otherwise. */booldrm_gpuvm_bo_put_deferred(struct drm_gpuvm_bo *vm_bo){ if (!vm_bo) return false;
drm_WARN_ON(vm_bo->vm->drm, !drm_gpuvm_immediate_mode(vm_bo->vm));
return !!kref_put_mutex(&vm_bo->kref, drm_gpuvm_bo_defer_free, &vm_bo->obj->gpuva.lock);}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put_deferred);
/** * drm_gpuvm_bo_deferred_cleanup() - clean up BOs in the deferred list * deferred cleanup * @gpuvm: the VM to clean up * * Cleans up &drm_gpuvm_bo instances in the deferred cleanup list. */voiddrm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm){ struct drm_gpuvm_bo *vm_bo; struct llist_node *bo_defer;
bo_defer = llist_del_all(&gpuvm->bo_defer); if (!bo_defer) return;
if (drm_gpuvm_resv_protected(gpuvm)) { dma_resv_lock(drm_gpuvm_resv(gpuvm), NULL); llist_for_each_entry(vm_bo, bo_defer, list.entry.bo_defer) { drm_gpuvm_bo_list_del(vm_bo, extobj, false); drm_gpuvm_bo_list_del(vm_bo, evict, false); } dma_resv_unlock(drm_gpuvm_resv(gpuvm)); }
while (bo_defer) { vm_bo = llist_entry(bo_defer, struct drm_gpuvm_bo, list.entry.bo_defer); bo_defer = bo_defer->next; drm_gpuvm_bo_destroy_not_in_lists(vm_bo); }}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_deferred_cleanup);
static struct drm_gpuvm_bo *__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj){ struct drm_gpuvm_bo *vm_bo;
drm_gem_gpuva_assert_lock_held(gpuvm, obj); drm_gem_for_each_gpuvm_bo(vm_bo, obj) if (vm_bo->vm == gpuvm) return vm_bo;
return NULL;}
/** * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given * &drm_gpuvm and &drm_gem_object * @gpuvm: The &drm_gpuvm the @obj is mapped in. * @obj: The &drm_gem_object being mapped in the @gpuvm. * * Find the &drm_gpuvm_bo representing the combination of the given * &drm_gpuvm and &drm_gem_object. If found, increases the reference * count of the &drm_gpuvm_bo accordingly. * * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure */struct drm_gpuvm_bo *drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj){ struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);
return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);
/** * drm_gpuvm_bo_obtain_locked() - obtains an instance of the &drm_gpuvm_bo for * the given &drm_gpuvm and &drm_gem_object * @gpuvm: The &drm_gpuvm the @obj is mapped in. * @obj: The &drm_gem_object being mapped in the @gpuvm. * * Find the &drm_gpuvm_bo representing the combination of the given * &drm_gpuvm and &drm_gem_object. If found, increases the reference * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new * &drm_gpuvm_bo. * * Requires the lock for the GEMs gpuva list. * * A new &drm_gpuvm_bo is added to the GEMs gpuva list. * * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure */struct drm_gpuvm_bo *drm_gpuvm_bo_obtain_locked(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj){ struct drm_gpuvm_bo *vm_bo;
/* * In immediate mode this would require the caller to hold the GEMs * gpuva mutex, but it's not okay to allocate while holding that lock, * and this method allocates. Immediate mode drivers should use * drm_gpuvm_bo_obtain_prealloc() instead. */ drm_WARN_ON(gpuvm->drm, drm_gpuvm_immediate_mode(gpuvm));
vm_bo = drm_gpuvm_bo_find(gpuvm, obj); if (vm_bo) return vm_bo;
vm_bo = drm_gpuvm_bo_create(gpuvm, obj); if (!vm_bo) return ERR_PTR(-ENOMEM);
drm_gem_gpuva_assert_lock_held(gpuvm, obj); list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list);
return vm_bo;}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_locked);
/** * drm_gpuvm_bo_obtain_prealloc() - obtains an instance of the &drm_gpuvm_bo * for the given &drm_gpuvm and &drm_gem_object * @__vm_bo: A pre-allocated struct drm_gpuvm_bo. * * Find the &drm_gpuvm_bo representing the combination of the given * &drm_gpuvm and &drm_gem_object. If found, increases the reference * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference * count is decreased. If not found @__vm_bo is returned without further * increase of the reference count. * * The provided @__vm_bo must not already be in the gpuva, evict, or extobj * lists prior to calling this method. * * A new &drm_gpuvm_bo is added to the GEMs gpuva list. * * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing * &drm_gpuvm_bo was found */struct drm_gpuvm_bo *drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo){ struct drm_gpuvm *gpuvm = __vm_bo->vm; struct drm_gem_object *obj = __vm_bo->obj; struct drm_gpuvm_bo *vm_bo;
drm_WARN_ON(gpuvm->drm, !drm_gpuvm_immediate_mode(gpuvm));
mutex_lock(&obj->gpuva.lock); vm_bo = drm_gpuvm_bo_find(gpuvm, obj); if (vm_bo) { mutex_unlock(&obj->gpuva.lock); kref_put(&__vm_bo->kref, drm_gpuvm_bo_destroy_not_in_lists_kref); return vm_bo; }
drm_gem_gpuva_assert_lock_held(gpuvm, obj); list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list); mutex_unlock(&obj->gpuva.lock);
return __vm_bo;}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
/** * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's * extobj list * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list. * * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list * already and if the corresponding &drm_gem_object is an external object, * actually. */voiddrm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo){ struct drm_gpuvm *gpuvm = vm_bo->vm; bool lock = !drm_gpuvm_resv_protected(gpuvm);
if (!lock) drm_gpuvm_resv_assert_held(gpuvm);
if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj)) drm_gpuvm_bo_list_add(vm_bo, extobj, lock);}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);
/** * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms * evicted list * @vm_bo: the &drm_gpuvm_bo to add or remove * @evict: indicates whether the object is evicted * * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvm's evicted list. */voiddrm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict){ struct drm_gpuvm *gpuvm = vm_bo->vm; struct drm_gem_object *obj = vm_bo->obj; bool lock = !drm_gpuvm_resv_protected(gpuvm);
dma_resv_assert_held(obj->resv); vm_bo->evicted = evict;
/* Can't add external objects to the evicted list directly if not using * internal spinlocks, since in this case the evicted list is protected * with the VM's common dma-resv lock. */ if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock) return;
if (evict) drm_gpuvm_bo_list_add(vm_bo, evict, lock); else drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);
static int__drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va){ struct rb_node *node; struct list_head *head;
if (drm_gpuva_it_iter_first(&gpuvm->rb.tree, GPUVA_START(va), GPUVA_LAST(va))) return -EEXIST;
va->vm = gpuvm;
drm_gpuva_it_insert(va, &gpuvm->rb.tree);
node = rb_prev(&va->rb.node); if (node) head = &(to_drm_gpuva(node))->rb.entry; else head = &gpuvm->rb.list;
list_add(&va->rb.entry, head);
return 0;}
/** * drm_gpuva_insert() - insert a &drm_gpuva * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in * @va: the &drm_gpuva to insert * * Insert a &drm_gpuva with a given address and range into a * &drm_gpuvm. * * It is safe to use this function using the safe versions of iterating the GPU * VA space, such as drm_gpuvm_for_each_va_safe() and * drm_gpuvm_for_each_va_range_safe(). * * Returns: 0 on success, negative error code on failure. */intdrm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va){ u64 addr = va->va.addr; u64 range = va->va.range; int ret;
if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range))) return -EINVAL;
ret = __drm_gpuva_insert(gpuvm, va); if (likely(!ret)) /* Take a reference of the GPUVM for the successfully inserted * drm_gpuva. We can't take the reference in * __drm_gpuva_insert() itself, since we don't want to increse * the reference count for the GPUVM's kernel_alloc_node. */ drm_gpuvm_get(gpuvm);
return ret;}EXPORT_SYMBOL_GPL(drm_gpuva_insert);
static void__drm_gpuva_remove(struct drm_gpuva *va){ drm_gpuva_it_remove(va, &va->vm->rb.tree); list_del_init(&va->rb.entry);}
/** * drm_gpuva_remove() - remove a &drm_gpuva * @va: the &drm_gpuva to remove * * This removes the given &va from the underlying tree. * * It is safe to use this function using the safe versions of iterating the GPU * VA space, such as drm_gpuvm_for_each_va_safe() and * drm_gpuvm_for_each_va_range_safe(). */voiddrm_gpuva_remove(struct drm_gpuva *va){ struct drm_gpuvm *gpuvm = va->vm;
if (unlikely(va == &gpuvm->kernel_alloc_node)) { drm_WARN(gpuvm->drm, 1, "Can't destroy kernel reserved node.\n"); return; }
__drm_gpuva_remove(va); drm_gpuvm_put(va->vm);}EXPORT_SYMBOL_GPL(drm_gpuva_remove);
/** * drm_gpuva_link() - link a &drm_gpuva * @va: the &drm_gpuva to link * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to * * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the * &drm_gpuvm_bo to the &drm_gem_object it is associated with. * * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional * reference of the latter is taken. * * This function expects the caller to protect the GEM's GPUVA list against * concurrent access using either the GEM's dma-resv or gpuva.lock mutex. */voiddrm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo){ struct drm_gem_object *obj = va->gem.obj; struct drm_gpuvm *gpuvm = va->vm;
if (unlikely(!obj)) return;
drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);
va->vm_bo = drm_gpuvm_bo_get(vm_bo);
drm_gem_gpuva_assert_lock_held(gpuvm, obj); list_add_tail(&va->gem.entry, &vm_bo->list.gpuva);}EXPORT_SYMBOL_GPL(drm_gpuva_link);
/** * drm_gpuva_unlink() - unlink a &drm_gpuva * @va: the &drm_gpuva to unlink * * This removes the given &va from the GPU VA list of the &drm_gem_object it is * associated with. * * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo. * * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of * the latter is dropped. * * This function expects the caller to protect the GEM's GPUVA list against * concurrent access using either the GEM's dma-resv or gpuva.lock mutex. */voiddrm_gpuva_unlink(struct drm_gpuva *va){ struct drm_gem_object *obj = va->gem.obj; struct drm_gpuvm_bo *vm_bo = va->vm_bo;
if (unlikely(!obj)) return;
drm_gem_gpuva_assert_lock_held(va->vm, obj); list_del_init(&va->gem.entry);
va->vm_bo = NULL; drm_gpuvm_bo_put(vm_bo);}EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
/** * drm_gpuva_unlink_defer() - unlink a &drm_gpuva with deferred vm_bo cleanup * @va: the &drm_gpuva to unlink * * Similar to drm_gpuva_unlink(), but uses drm_gpuvm_bo_put_deferred() and takes * the lock for the caller. */voiddrm_gpuva_unlink_defer(struct drm_gpuva *va){ struct drm_gem_object *obj = va->gem.obj; struct drm_gpuvm_bo *vm_bo = va->vm_bo; bool should_defer_bo;
if (unlikely(!obj)) return;
drm_WARN_ON(vm_bo->vm->drm, !drm_gpuvm_immediate_mode(vm_bo->vm));
mutex_lock(&obj->gpuva.lock); list_del_init(&va->gem.entry);
/* * This is drm_gpuvm_bo_put_deferred() except we already hold the mutex. */ should_defer_bo = kref_put(&vm_bo->kref, drm_gpuvm_bo_into_zombie); mutex_unlock(&obj->gpuva.lock); if (should_defer_bo) drm_gpuvm_bo_defer_zombie_cleanup(vm_bo);
va->vm_bo = NULL;}EXPORT_SYMBOL_GPL(drm_gpuva_unlink_defer);
/** * drm_gpuva_find_first() - find the first &drm_gpuva in the given range * @gpuvm: the &drm_gpuvm to search in * @addr: the &drm_gpuvas address * @range: the &drm_gpuvas range * * Returns: the first &drm_gpuva within the given range */struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ u64 last = addr + range - 1;
return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last);}EXPORT_SYMBOL_GPL(drm_gpuva_find_first);
/** * drm_gpuva_find() - find a &drm_gpuva * @gpuvm: the &drm_gpuvm to search in * @addr: the &drm_gpuvas address * @range: the &drm_gpuvas range * * Returns: the &drm_gpuva at a given &addr and with a given &range */struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ struct drm_gpuva *va;
va = drm_gpuva_find_first(gpuvm, addr, range); if (!va) goto out;
if (va->va.addr != addr || va->va.range != range) goto out;
return va;
out: return NULL;}EXPORT_SYMBOL_GPL(drm_gpuva_find);
/** * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space * is empty * @gpuvm: the &drm_gpuvm to check the range for * @addr: the start address of the range * @range: the range of the interval * * Returns: true if the interval is empty, false otherwise */booldrm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ return !drm_gpuva_find_first(gpuvm, addr, range);}EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty);
/** * drm_gpuva_map() - helper to insert a &drm_gpuva according to a * &drm_gpuva_op_map * @gpuvm: the &drm_gpuvm * @va: the &drm_gpuva to insert * @op: the &drm_gpuva_op_map to initialize @va with * * Initializes the @va from the @op and inserts it into the given @gpuvm. */voiddrm_gpuva_map(struct drm_gpuvm *gpuvm, struct drm_gpuva *va, const struct drm_gpuva_op_map *op){ drm_gpuva_init_from_op(va, op); drm_gpuva_insert(gpuvm, va);}EXPORT_SYMBOL_GPL(drm_gpuva_map);
/** * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a * &drm_gpuva_op_remap * @prev: the &drm_gpuva to remap when keeping the start of a mapping * @next: the &drm_gpuva to remap when keeping the end of a mapping * @op: the &drm_gpuva_op_remap to initialize @prev and @next with * * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or * @next. */voiddrm_gpuva_remap(struct drm_gpuva *prev, struct drm_gpuva *next, const struct drm_gpuva_op_remap *op){ struct drm_gpuva *va = op->unmap->va; struct drm_gpuvm *gpuvm = va->vm;
drm_gpuva_remove(va);
if (op->prev) { drm_gpuva_init_from_op(prev, op->prev); drm_gpuva_insert(gpuvm, prev); }
if (op->next) { drm_gpuva_init_from_op(next, op->next); drm_gpuva_insert(gpuvm, next); }}EXPORT_SYMBOL_GPL(drm_gpuva_remap);
/** * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a * &drm_gpuva_op_unmap * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove * * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap. */voiddrm_gpuva_unmap(const struct drm_gpuva_op_unmap *op){ drm_gpuva_remove(op->va);}EXPORT_SYMBOL_GPL(drm_gpuva_unmap);
static intop_map_cb(const struct drm_gpuvm_ops *fn, void *priv, const struct drm_gpuvm_map_req *req){ struct drm_gpuva_op op = {};
if (!req) return 0;
op.op = DRM_GPUVA_OP_MAP; op.map.va.addr = req->map.va.addr; op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset;
return fn->sm_step_map(&op, priv);}
static intop_remap_cb(const struct drm_gpuvm_ops *fn, void *priv, struct drm_gpuva_op_map *prev, struct drm_gpuva_op_map *next, struct drm_gpuva_op_unmap *unmap){ struct drm_gpuva_op op = {}; struct drm_gpuva_op_remap *r;
op.op = DRM_GPUVA_OP_REMAP; r = &op.remap; r->prev = prev; r->next = next; r->unmap = unmap;
return fn->sm_step_remap(&op, priv);}
static intop_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, struct drm_gpuva *va, bool merge, bool madvise){ struct drm_gpuva_op op = {};
if (madvise) return 0;
op.op = DRM_GPUVA_OP_UNMAP; op.unmap.va = va; op.unmap.keep = merge;
return fn->sm_step_unmap(&op, priv);}
static int__drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, const struct drm_gpuvm_map_req *req, bool madvise){ struct drm_gem_object *req_obj = req->map.gem.obj; const struct drm_gpuvm_map_req *op_map = madvise ? NULL : req; struct drm_gpuva *va, *next; u64 req_offset = req->map.gem.offset; u64 req_range = req->map.va.range; u64 req_addr = req->map.va.addr; u64 req_end = req_addr + req_range; int ret;
if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) return -EINVAL;
drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { struct drm_gem_object *obj = va->gem.obj; u64 offset = va->gem.offset; u64 addr = va->va.addr; u64 range = va->va.range; u64 end = addr + range; bool merge = !!va->gem.obj;
if (madvise && obj) continue;
if (addr == req_addr) { merge &= obj == req_obj && offset == req_offset;
if (end == req_end) { ret = op_unmap_cb(ops, priv, va, merge, madvise); if (ret) return ret; break; }
if (end < req_end) { ret = op_unmap_cb(ops, priv, va, merge, madvise); if (ret) return ret; continue; }
if (end > req_end) { struct drm_gpuva_op_map n = { .va.addr = req_end, .va.range = range - req_range, .gem.obj = obj, .gem.offset = offset + req_range, }; struct drm_gpuva_op_unmap u = { .va = va, .keep = merge, };
ret = op_remap_cb(ops, priv, NULL, &n, &u); if (ret) return ret;
if (madvise) op_map = req; break; } } else if (addr < req_addr) { u64 ls_range = req_addr - addr; struct drm_gpuva_op_map p = { .va.addr = addr, .va.range = ls_range, .gem.obj = obj, .gem.offset = offset, }; struct drm_gpuva_op_unmap u = { .va = va };
merge &= obj == req_obj && offset + ls_range == req_offset; u.keep = merge;
if (end == req_end) { ret = op_remap_cb(ops, priv, &p, NULL, &u); if (ret) return ret;
if (madvise) op_map = req; break; }
if (end < req_end) { ret = op_remap_cb(ops, priv, &p, NULL, &u); if (ret) return ret;
if (madvise) { struct drm_gpuvm_map_req map_req = { .map.va.addr = req_addr, .map.va.range = end - req_addr, };
ret = op_map_cb(ops, priv, &map_req); if (ret) return ret; }
continue; }
if (end > req_end) { struct drm_gpuva_op_map n = { .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, .gem.offset = offset + ls_range + req_range, };
ret = op_remap_cb(ops, priv, &p, &n, &u); if (ret) return ret;
if (madvise) op_map = req; break; } } else if (addr > req_addr) { merge &= obj == req_obj && offset == req_offset + (addr - req_addr);
if (end == req_end) { ret = op_unmap_cb(ops, priv, va, merge, madvise); if (ret) return ret;
break; }
if (end < req_end) { ret = op_unmap_cb(ops, priv, va, merge, madvise); if (ret) return ret;
continue; }
if (end > req_end) { struct drm_gpuva_op_map n = { .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, .gem.offset = offset + req_end - addr, }; struct drm_gpuva_op_unmap u = { .va = va, .keep = merge, };
ret = op_remap_cb(ops, priv, NULL, &n, &u); if (ret) return ret;
if (madvise) { struct drm_gpuvm_map_req map_req = { .map.va.addr = addr, .map.va.range = req_end - addr, };
return op_map_cb(ops, priv, &map_req); } break; } } } return op_map_cb(ops, priv, op_map);}
static int__drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, u64 req_addr, u64 req_range){ struct drm_gpuva *va, *next; u64 req_end = req_addr + req_range; int ret;
if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) return -EINVAL;
drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { struct drm_gpuva_op_map prev = {}, next = {}; bool prev_split = false, next_split = false; struct drm_gem_object *obj = va->gem.obj; u64 offset = va->gem.offset; u64 addr = va->va.addr; u64 range = va->va.range; u64 end = addr + range;
if (addr < req_addr) { prev.va.addr = addr; prev.va.range = req_addr - addr; prev.gem.obj = obj; prev.gem.offset = offset;
prev_split = true; }
if (end > req_end) { next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; next.gem.offset = offset + (req_end - addr);
next_split = true; }
if (prev_split || next_split) { struct drm_gpuva_op_unmap unmap = { .va = va };
ret = op_remap_cb(ops, priv, prev_split ? &prev : NULL, next_split ? &next : NULL, &unmap); if (ret) return ret; } else { ret = op_unmap_cb(ops, priv, va, false, false); if (ret) return ret; } }
return 0;}
/** * drm_gpuvm_sm_map() - calls the &drm_gpuva_op split/merge steps * @gpuvm: the &drm_gpuvm representing the GPU VA space * @priv: pointer to a driver private data structure * @req: ptr to struct drm_gpuvm_map_req * * This function iterates the given range of the GPU VA space. It utilizes the * &drm_gpuvm_ops to call back into the driver providing the split and merge * steps. * * Drivers may use these callbacks to update the GPU VA space right away within * the callback. In case the driver decides to copy and store the operations for * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to * be called before the &drm_gpuvm's view of the GPU VA space was * updated with the previous set of operations. To update the * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * A sequence of callbacks can contain map, unmap and remap operations, but * the sequence of callbacks might also be empty if no operation is required, * e.g. if the requested mapping already exists in the exact same way. * * There can be an arbitrary amount of unmap operations, a maximum of two remap * operations and a single map operation. The latter one represents the original * map operation requested by the caller. * * Returns: 0 on success or a negative error code */intdrm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, const struct drm_gpuvm_map_req *req){ const struct drm_gpuvm_ops *ops = gpuvm->ops;
if (unlikely(!(ops && ops->sm_step_map && ops->sm_step_remap && ops->sm_step_unmap))) return -EINVAL;
return __drm_gpuvm_sm_map(gpuvm, ops, priv, req, false);}EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map);
/** * drm_gpuvm_sm_unmap() - calls the &drm_gpuva_ops to split on unmap * @gpuvm: the &drm_gpuvm representing the GPU VA space * @priv: pointer to a driver private data structure * @req_addr: the start address of the range to unmap * @req_range: the range of the mappings to unmap * * This function iterates the given range of the GPU VA space. It utilizes the * &drm_gpuvm_ops to call back into the driver providing the operations to * unmap and, if required, split existing mappings. * * Drivers may use these callbacks to update the GPU VA space right away within * the callback. In case the driver decides to copy and store the operations for * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be * called before the &drm_gpuvm's view of the GPU VA space was updated * with the previous set of operations. To update the &drm_gpuvm's view * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or * drm_gpuva_destroy_unlocked() should be used. * * A sequence of callbacks can contain unmap and remap operations, depending on * whether there are actual overlapping mappings to split. * * There can be an arbitrary amount of unmap operations and a maximum of two * remap operations. * * Returns: 0 on success or a negative error code */intdrm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, u64 req_addr, u64 req_range){ const struct drm_gpuvm_ops *ops = gpuvm->ops;
if (unlikely(!(ops && ops->sm_step_remap && ops->sm_step_unmap))) return -EINVAL;
return __drm_gpuvm_sm_unmap(gpuvm, ops, priv, req_addr, req_range);}EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap);
static intdrm_gpuva_sm_step_lock(struct drm_gpuva_op *op, void *priv){ struct drm_exec *exec = priv;
switch (op->op) { case DRM_GPUVA_OP_REMAP: if (op->remap.unmap->va->gem.obj) return drm_exec_lock_obj(exec, op->remap.unmap->va->gem.obj); return 0; case DRM_GPUVA_OP_UNMAP: if (op->unmap.va->gem.obj) return drm_exec_lock_obj(exec, op->unmap.va->gem.obj); return 0; default: return 0; }}
static const struct drm_gpuvm_ops lock_ops = { .sm_step_map = drm_gpuva_sm_step_lock, .sm_step_remap = drm_gpuva_sm_step_lock, .sm_step_unmap = drm_gpuva_sm_step_lock,};
/** * drm_gpuvm_sm_map_exec_lock() - locks the objects touched by a drm_gpuvm_sm_map() * @gpuvm: the &drm_gpuvm representing the GPU VA space * @exec: the &drm_exec locking context * @num_fences: for newly mapped objects, the # of fences to reserve * @req: ptr to drm_gpuvm_map_req struct * * This function locks (drm_exec_lock_obj()) objects that will be unmapped/ * remapped, and locks+prepares (drm_exec_prepare_object()) objects that * will be newly mapped. * * The expected usage is:: * * vm_bind { * struct drm_exec exec; * * // IGNORE_DUPLICATES is required, INTERRUPTIBLE_WAIT is recommended: * drm_exec_init(&exec, IGNORE_DUPLICATES | INTERRUPTIBLE_WAIT, 0); * * drm_exec_until_all_locked (&exec) { * for_each_vm_bind_operation { * switch (op->op) { * case DRIVER_OP_UNMAP: * ret = drm_gpuvm_sm_unmap_exec_lock(gpuvm, &exec, op->addr, op->range); * break; * case DRIVER_OP_MAP: * ret = drm_gpuvm_sm_map_exec_lock(gpuvm, &exec, num_fences, &req); * break; * } * * drm_exec_retry_on_contention(&exec); * if (ret) * return ret; * } * } * } * * This enables all locking to be performed before the driver begins modifying * the VM. This is safe to do in the case of overlapping DRIVER_VM_BIND_OPs, * where an earlier op can alter the sequence of steps generated for a later * op, because the later altered step will involve the same GEM object(s) * already seen in the earlier locking step. For example: * * 1) An earlier driver DRIVER_OP_UNMAP op removes the need for a * DRM_GPUVA_OP_REMAP/UNMAP step. This is safe because we've already * locked the GEM object in the earlier DRIVER_OP_UNMAP op. * * 2) An earlier DRIVER_OP_MAP op overlaps with a later DRIVER_OP_MAP/UNMAP * op, introducing a DRM_GPUVA_OP_REMAP/UNMAP that wouldn't have been * required without the earlier DRIVER_OP_MAP. This is safe because we've * already locked the GEM object in the earlier DRIVER_OP_MAP step. * * Returns: 0 on success or a negative error code */intdrm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences, struct drm_gpuvm_map_req *req){ struct drm_gem_object *req_obj = req->map.gem.obj;
if (req_obj) { int ret = drm_exec_prepare_obj(exec, req_obj, num_fences); if (ret) return ret; }
return __drm_gpuvm_sm_map(gpuvm, &lock_ops, exec, req, false);
}EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_exec_lock);
/** * drm_gpuvm_sm_unmap_exec_lock() - locks the objects touched by drm_gpuvm_sm_unmap() * @gpuvm: the &drm_gpuvm representing the GPU VA space * @exec: the &drm_exec locking context * @req_addr: the start address of the range to unmap * @req_range: the range of the mappings to unmap * * This function locks (drm_exec_lock_obj()) objects that will be unmapped/ * remapped by drm_gpuvm_sm_unmap(). * * See drm_gpuvm_sm_map_exec_lock() for expected usage. * * Returns: 0 on success or a negative error code */intdrm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, u64 req_addr, u64 req_range){ return __drm_gpuvm_sm_unmap(gpuvm, &lock_ops, exec, req_addr, req_range);}EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_exec_lock);
static struct drm_gpuva_op *gpuva_op_alloc(struct drm_gpuvm *gpuvm){ const struct drm_gpuvm_ops *fn = gpuvm->ops; struct drm_gpuva_op *op;
if (fn && fn->op_alloc) op = fn->op_alloc(); else op = kzalloc_obj(*op);
if (unlikely(!op)) return NULL;
return op;}
static voidgpuva_op_free(struct drm_gpuvm *gpuvm, struct drm_gpuva_op *op){ const struct drm_gpuvm_ops *fn = gpuvm->ops;
if (fn && fn->op_free) fn->op_free(op); else kfree(op);}
static intdrm_gpuva_sm_step(struct drm_gpuva_op *__op, void *priv){ struct { struct drm_gpuvm *vm; struct drm_gpuva_ops *ops; } *args = priv; struct drm_gpuvm *gpuvm = args->vm; struct drm_gpuva_ops *ops = args->ops; struct drm_gpuva_op *op;
op = gpuva_op_alloc(gpuvm); if (unlikely(!op)) goto err;
memcpy(op, __op, sizeof(*op));
if (op->op == DRM_GPUVA_OP_REMAP) { struct drm_gpuva_op_remap *__r = &__op->remap; struct drm_gpuva_op_remap *r = &op->remap;
r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap), GFP_KERNEL); if (unlikely(!r->unmap)) goto err_free_op;
if (__r->prev) { r->prev = kmemdup(__r->prev, sizeof(*r->prev), GFP_KERNEL); if (unlikely(!r->prev)) goto err_free_unmap; }
if (__r->next) { r->next = kmemdup(__r->next, sizeof(*r->next), GFP_KERNEL); if (unlikely(!r->next)) goto err_free_prev; } }
list_add_tail(&op->entry, &ops->list);
return 0;
err_free_unmap: kfree(op->remap.unmap);err_free_prev: kfree(op->remap.prev);err_free_op: gpuva_op_free(gpuvm, op);err: return -ENOMEM;}
static const struct drm_gpuvm_ops gpuvm_list_ops = { .sm_step_map = drm_gpuva_sm_step, .sm_step_remap = drm_gpuva_sm_step, .sm_step_unmap = drm_gpuva_sm_step,};
static struct drm_gpuva_ops *__drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_map_req *req, bool madvise){ struct drm_gpuva_ops *ops; struct { struct drm_gpuvm *vm; struct drm_gpuva_ops *ops; } args; int ret;
ops = kzalloc_obj(*ops); if (unlikely(!ops)) return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&ops->list);
args.vm = gpuvm; args.ops = ops;
ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args, req, madvise); if (ret) goto err_free_ops;
return ops;
err_free_ops: drm_gpuva_ops_free(gpuvm, ops); return ERR_PTR(ret);}
/** * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge * @gpuvm: the &drm_gpuvm representing the GPU VA space * @req: map request arguments * * This function creates a list of operations to perform splitting and merging * of existing mapping(s) with the newly requested one. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain map, unmap and remap operations, but it * also can be empty if no operation is required, e.g. if the requested mapping * already exists in the exact same way. * * There can be an arbitrary amount of unmap operations, a maximum of two remap * operations and a single map operation. The latter one represents the original * map operation requested by the caller. * * Note that before calling this function again with another mapping request it * is necessary to update the &drm_gpuvm's view of the GPU VA space. The * previously obtained operations must be either processed or abandoned. To * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */struct drm_gpuva_ops *drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_map_req *req){ return __drm_gpuvm_sm_map_ops_create(gpuvm, req, false);}EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create);
/** * drm_gpuvm_madvise_ops_create() - creates the &drm_gpuva_ops to split * @gpuvm: the &drm_gpuvm representing the GPU VA space * @req: map request arguments * * This function creates a list of operations to perform splitting * of existent mapping(s) at start or end, based on the request map. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain map and remap operations, but it * also can be empty if no operation is required, e.g. if the requested mapping * already exists is the exact same way. * * There will be no unmap operations, a maximum of two remap operations and two * map operations. The two map operations correspond to: one from start to the * end of drm_gpuvaX, and another from the start of drm_gpuvaY to end. * * Note that before calling this function again with another mapping request it * is necessary to update the &drm_gpuvm's view of the GPU VA space. The * previously obtained operations must be either processed or abandoned. To * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */struct drm_gpuva_ops *drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_map_req *req){ return __drm_gpuvm_sm_map_ops_create(gpuvm, req, true);}EXPORT_SYMBOL_GPL(drm_gpuvm_madvise_ops_create);
/** * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on * unmap * @gpuvm: the &drm_gpuvm representing the GPU VA space * @req_addr: the start address of the range to unmap * @req_range: the range of the mappings to unmap * * This function creates a list of operations to perform unmapping and, if * required, splitting of the mappings overlapping the unmap range. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain unmap and remap operations, depending on * whether there are actual overlapping mappings to split. * * There can be an arbitrary amount of unmap operations and a maximum of two * remap operations. * * Note that before calling this function again with another range to unmap it * is necessary to update the &drm_gpuvm's view of the GPU VA space. The * previously obtained operations must be processed or abandoned. To update the * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be * used. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */struct drm_gpuva_ops *drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, u64 req_addr, u64 req_range){ struct drm_gpuva_ops *ops; struct { struct drm_gpuvm *vm; struct drm_gpuva_ops *ops; } args; int ret;
ops = kzalloc_obj(*ops); if (unlikely(!ops)) return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&ops->list);
args.vm = gpuvm; args.ops = ops;
ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args, req_addr, req_range); if (ret) goto err_free_ops;
return ops;
err_free_ops: drm_gpuva_ops_free(gpuvm, ops); return ERR_PTR(ret);}EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create);
/** * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch * @gpuvm: the &drm_gpuvm representing the GPU VA space * @addr: the start address of the range to prefetch * @range: the range of the mappings to prefetch * * This function creates a list of operations to perform prefetching. * * The list can be iterated with &drm_gpuva_for_each_op and must be processed * in the given order. It can contain prefetch operations. * * There can be an arbitrary amount of prefetch operations. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */struct drm_gpuva_ops *drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, u64 addr, u64 range){ struct drm_gpuva_ops *ops; struct drm_gpuva_op *op; struct drm_gpuva *va; u64 end = addr + range; int ret;
ops = kzalloc_obj(*ops); if (!ops) return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&ops->list);
drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { op = gpuva_op_alloc(gpuvm); if (!op) { ret = -ENOMEM; goto err_free_ops; }
op->op = DRM_GPUVA_OP_PREFETCH; op->prefetch.va = va; list_add_tail(&op->entry, &ops->list); }
return ops;
err_free_ops: drm_gpuva_ops_free(gpuvm, ops); return ERR_PTR(ret);}EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
/** * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM * @vm_bo: the &drm_gpuvm_bo abstraction * * This function creates a list of operations to perform unmapping for every * GPUVA attached to a GEM. * * The list can be iterated with &drm_gpuva_for_each_op and consists out of an * arbitrary amount of unmap operations. * * After the caller finished processing the returned &drm_gpuva_ops, they must * be freed with &drm_gpuva_ops_free. * * This function expects the caller to protect the GEM's GPUVA list against * concurrent access using either the GEM's dma-resv or gpuva.lock mutex. * * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure */struct drm_gpuva_ops *drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo){ struct drm_gpuva_ops *ops; struct drm_gpuva_op *op; struct drm_gpuva *va; int ret;
drm_gem_gpuva_assert_lock_held(vm_bo->vm, vm_bo->obj);
ops = kzalloc_obj(*ops); if (!ops) return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&ops->list);
drm_gpuvm_bo_for_each_va(va, vm_bo) { op = gpuva_op_alloc(vm_bo->vm); if (!op) { ret = -ENOMEM; goto err_free_ops; }
op->op = DRM_GPUVA_OP_UNMAP; op->unmap.va = va; list_add_tail(&op->entry, &ops->list); }
return ops;
err_free_ops: drm_gpuva_ops_free(vm_bo->vm, ops); return ERR_PTR(ret);}EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);
/** * drm_gpuva_ops_free() - free the given &drm_gpuva_ops * @gpuvm: the &drm_gpuvm the ops were created for * @ops: the &drm_gpuva_ops to free * * Frees the given &drm_gpuva_ops structure including all the ops associated * with it. */voiddrm_gpuva_ops_free(struct drm_gpuvm *gpuvm, struct drm_gpuva_ops *ops){ struct drm_gpuva_op *op, *next;
drm_gpuva_for_each_op_safe(op, next, ops) { list_del(&op->entry);
if (op->op == DRM_GPUVA_OP_REMAP) { kfree(op->remap.prev); kfree(op->remap.next); kfree(op->remap.unmap); }
gpuva_op_free(gpuvm, op); }
kfree(ops);}EXPORT_SYMBOL_GPL(drm_gpuva_ops_free);
MODULE_DESCRIPTION("DRM GPUVM");MODULE_LICENSE("GPL");