Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450// SPDX-License-Identifier: GPL-2.0-only/* Copyright (C) 2009 Red Hat, Inc. * Copyright (C) 2006 Rusty Russell IBM Corporation * * Author: Michael S. Tsirkin <mst@redhat.com> * * Inspiration, some code, and most witty comments come from * Documentation/virtual/lguest/lguest.c, by Rusty Russell * * Generic code for virtio server in host kernel. */
#include <linux/eventfd.h>#include <linux/vhost.h>#include <linux/uio.h>#include <linux/mm.h>#include <linux/miscdevice.h>#include <linux/mutex.h>#include <linux/poll.h>#include <linux/file.h>#include <linux/highmem.h>#include <linux/slab.h>#include <linux/vmalloc.h>#include <linux/kthread.h>#include <linux/cgroup.h>#include <linux/module.h>#include <linux/sort.h>#include <linux/sched/mm.h>#include <linux/sched/signal.h>#include <linux/sched/vhost_task.h>#include <linux/interval_tree_generic.h>#include <linux/nospec.h>#include <linux/kcov.h>
#include "vhost.h"
static ushort max_mem_regions = 64;module_param(max_mem_regions, ushort, 0444);MODULE_PARM_DESC(max_mem_regions, "Maximum number of memory regions in memory map. (default: 64)");static int max_iotlb_entries = 2048;module_param(max_iotlb_entries, int, 0444);MODULE_PARM_DESC(max_iotlb_entries, "Maximum number of iotlb entries. (default: 2048)");static bool fork_from_owner_default = VHOST_FORK_OWNER_TASK;
#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROLmodule_param(fork_from_owner_default, bool, 0444);MODULE_PARM_DESC(fork_from_owner_default, "Set task mode as the default(default: Y)");#endif
enum { VHOST_MEMORY_F_LOG = 0x1,};
#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACYstatic void vhost_disable_cross_endian(struct vhost_virtqueue *vq){ vq->user_be = !virtio_legacy_is_little_endian();}
static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq){ vq->user_be = true;}
static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq){ vq->user_be = false;}
static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp){ struct vhost_vring_state s;
if (vq->private_data) return -EBUSY;
if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT;
if (s.num != VHOST_VRING_LITTLE_ENDIAN && s.num != VHOST_VRING_BIG_ENDIAN) return -EINVAL;
if (s.num == VHOST_VRING_BIG_ENDIAN) vhost_enable_cross_endian_big(vq); else vhost_enable_cross_endian_little(vq);
return 0;}
static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, int __user *argp){ struct vhost_vring_state s = { .index = idx, .num = vq->user_be };
if (copy_to_user(argp, &s, sizeof(s))) return -EFAULT;
return 0;}
static void vhost_init_is_le(struct vhost_virtqueue *vq){ /* Note for legacy virtio: user_be is initialized at reset time * according to the host endianness. If userspace does not set an * explicit endianness, the default behavior is native endian, as * expected by legacy virtio. */ vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;}#elsestatic void vhost_disable_cross_endian(struct vhost_virtqueue *vq){}
static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp){ return -ENOIOCTLCMD;}
static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, int __user *argp){ return -ENOIOCTLCMD;}
static void vhost_init_is_le(struct vhost_virtqueue *vq){ vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || virtio_legacy_is_little_endian();}#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
static void vhost_reset_is_le(struct vhost_virtqueue *vq){ vhost_init_is_le(vq);}
struct vhost_flush_struct { struct vhost_work work; struct completion wait_event;};
static void vhost_flush_work(struct vhost_work *work){ struct vhost_flush_struct *s;
s = container_of(work, struct vhost_flush_struct, work); complete(&s->wait_event);}
static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, poll_table *pt){ struct vhost_poll *poll;
poll = container_of(pt, struct vhost_poll, table); poll->wqh = wqh; add_wait_queue(wqh, &poll->wait);}
static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key){ struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); struct vhost_work *work = &poll->work;
if (!(key_to_poll(key) & poll->mask)) return 0;
if (!poll->dev->use_worker) work->fn(work); else vhost_poll_queue(poll);
return 0;}
void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn){ clear_bit(VHOST_WORK_QUEUED, &work->flags); work->fn = fn;}EXPORT_SYMBOL_GPL(vhost_work_init);
/* Init poll structure */void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, __poll_t mask, struct vhost_dev *dev, struct vhost_virtqueue *vq){ init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); init_poll_funcptr(&poll->table, vhost_poll_func); poll->mask = mask; poll->dev = dev; poll->wqh = NULL; poll->vq = vq;
vhost_work_init(&poll->work, fn);}EXPORT_SYMBOL_GPL(vhost_poll_init);
/* Start polling a file. We add ourselves to file's wait queue. The caller must * keep a reference to a file until after vhost_poll_stop is called. */int vhost_poll_start(struct vhost_poll *poll, struct file *file){ __poll_t mask;
if (poll->wqh) return 0;
mask = vfs_poll(file, &poll->table); if (mask) vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask)); if (mask & EPOLLERR) { vhost_poll_stop(poll); return -EINVAL; }
return 0;}EXPORT_SYMBOL_GPL(vhost_poll_start);
/* Stop polling a file. After this function returns, it becomes safe to drop the * file reference. You must also flush afterwards. */void vhost_poll_stop(struct vhost_poll *poll){ if (poll->wqh) { remove_wait_queue(poll->wqh, &poll->wait); poll->wqh = NULL; }}EXPORT_SYMBOL_GPL(vhost_poll_stop);
static void vhost_worker_queue(struct vhost_worker *worker, struct vhost_work *work){ if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) { /* We can only add the work to the list after we're * sure it was not in the list. * test_and_set_bit() implies a memory barrier. */ llist_add(&work->node, &worker->work_list); worker->ops->wakeup(worker); }}
bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work){ struct vhost_worker *worker; bool queued = false;
rcu_read_lock(); worker = rcu_dereference(vq->worker); if (worker) { queued = true; vhost_worker_queue(worker, work); } rcu_read_unlock();
return queued;}EXPORT_SYMBOL_GPL(vhost_vq_work_queue);
/** * __vhost_worker_flush - flush a worker * @worker: worker to flush * * The worker's flush_mutex must be held. */static void __vhost_worker_flush(struct vhost_worker *worker){ struct vhost_flush_struct flush;
if (!worker->attachment_cnt || worker->killed) return;
init_completion(&flush.wait_event); vhost_work_init(&flush.work, vhost_flush_work);
vhost_worker_queue(worker, &flush.work); /* * Drop mutex in case our worker is killed and it needs to take the * mutex to force cleanup. */ mutex_unlock(&worker->mutex); wait_for_completion(&flush.wait_event); mutex_lock(&worker->mutex);}
static void vhost_worker_flush(struct vhost_worker *worker){ mutex_lock(&worker->mutex); __vhost_worker_flush(worker); mutex_unlock(&worker->mutex);}
void vhost_dev_flush(struct vhost_dev *dev){ struct vhost_worker *worker; unsigned long i;
xa_for_each(&dev->worker_xa, i, worker) vhost_worker_flush(worker);}EXPORT_SYMBOL_GPL(vhost_dev_flush);
/* A lockless hint for busy polling code to exit the loop */bool vhost_vq_has_work(struct vhost_virtqueue *vq){ struct vhost_worker *worker; bool has_work = false;
rcu_read_lock(); worker = rcu_dereference(vq->worker); if (worker && !llist_empty(&worker->work_list)) has_work = true; rcu_read_unlock();
return has_work;}EXPORT_SYMBOL_GPL(vhost_vq_has_work);
void vhost_poll_queue(struct vhost_poll *poll){ vhost_vq_work_queue(poll->vq, &poll->work);}EXPORT_SYMBOL_GPL(vhost_poll_queue);
static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq){ int j;
for (j = 0; j < VHOST_NUM_ADDRS; j++) vq->meta_iotlb[j] = NULL;}
/* Caller must hold the virtqueue mutex. */static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq){ vq->desc = NULL; vq->avail = NULL; vq->used = NULL; vq->log_used = false; vq->log_addr = -1ull; __vhost_vq_meta_reset(vq);}
static void vhost_vq_meta_reset(struct vhost_dev *d){ int i;
for (i = 0; i < d->nvqs; ++i) __vhost_vq_meta_reset(d->vqs[i]);}
static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx){ call_ctx->ctx = NULL; memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer));}
bool vhost_vq_is_setup(struct vhost_virtqueue *vq){ return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq);}EXPORT_SYMBOL_GPL(vhost_vq_is_setup);
static void vhost_vq_reset(struct vhost_dev *dev, struct vhost_virtqueue *vq){ vq->num = 1; vq->desc = NULL; vq->avail = NULL; vq->used = NULL; vq->last_avail_idx = 0; vq->next_avail_head = 0; vq->avail_idx = 0; vq->last_used_idx = 0; vq->signalled_used = 0; vq->signalled_used_valid = false; vq->used_flags = 0; vq->log_used = false; vq->log_addr = -1ull; vq->private_data = NULL; virtio_features_zero(vq->acked_features_array); vq->acked_backend_features = 0; vq->log_base = NULL; vq->error_ctx = NULL; vq->kick = NULL; vq->log_ctx = NULL; vhost_disable_cross_endian(vq); vhost_reset_is_le(vq); vq->busyloop_timeout = 0; vq->umem = NULL; vq->iotlb = NULL; vq->iotlb_miss = NULL; rcu_assign_pointer(vq->worker, NULL); vhost_vring_call_reset(&vq->call_ctx); __vhost_vq_meta_reset(vq);}
static int vhost_run_work_kthread_list(void *data){ struct vhost_worker *worker = data; struct vhost_work *work, *work_next; struct vhost_dev *dev = worker->dev; struct llist_node *node;
kthread_use_mm(dev->mm);
for (;;) { /* mb paired w/ kthread_stop */ set_current_state(TASK_INTERRUPTIBLE);
if (kthread_should_stop()) { __set_current_state(TASK_RUNNING); break; } node = llist_del_all(&worker->work_list); if (!node) schedule();
node = llist_reverse_order(node); /* make sure flag is seen after deletion */ smp_wmb(); llist_for_each_entry_safe(work, work_next, node, node) { clear_bit(VHOST_WORK_QUEUED, &work->flags); __set_current_state(TASK_RUNNING); kcov_remote_start_common(worker->kcov_handle); work->fn(work); kcov_remote_stop(); cond_resched(); } } kthread_unuse_mm(dev->mm);
return 0;}
static bool vhost_run_work_list(void *data){ struct vhost_worker *worker = data; struct vhost_work *work, *work_next; struct llist_node *node;
node = llist_del_all(&worker->work_list); if (node) { __set_current_state(TASK_RUNNING);
node = llist_reverse_order(node); /* make sure flag is seen after deletion */ smp_wmb(); llist_for_each_entry_safe(work, work_next, node, node) { clear_bit(VHOST_WORK_QUEUED, &work->flags); kcov_remote_start_common(worker->kcov_handle); work->fn(work); kcov_remote_stop(); cond_resched(); } }
return !!node;}
static void vhost_worker_killed(void *data){ struct vhost_worker *worker = data; struct vhost_dev *dev = worker->dev; struct vhost_virtqueue *vq; int i, attach_cnt = 0;
mutex_lock(&worker->mutex); worker->killed = true;
for (i = 0; i < dev->nvqs; i++) { vq = dev->vqs[i];
mutex_lock(&vq->mutex); if (worker == rcu_dereference_check(vq->worker, lockdep_is_held(&vq->mutex))) { rcu_assign_pointer(vq->worker, NULL); attach_cnt++; } mutex_unlock(&vq->mutex); }
worker->attachment_cnt -= attach_cnt; if (attach_cnt) synchronize_rcu(); /* * Finish vhost_worker_flush calls and any other works that snuck in * before the synchronize_rcu. */ vhost_run_work_list(worker); mutex_unlock(&worker->mutex);}
static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq){ kfree(vq->indirect); vq->indirect = NULL; kfree(vq->log); vq->log = NULL; kfree(vq->heads); vq->heads = NULL; kfree(vq->nheads); vq->nheads = NULL;}
/* Helper to allocate iovec buffers for all vqs. */static long vhost_dev_alloc_iovecs(struct vhost_dev *dev){ struct vhost_virtqueue *vq; int i;
for (i = 0; i < dev->nvqs; ++i) { vq = dev->vqs[i]; vq->indirect = kmalloc_objs(*vq->indirect, UIO_MAXIOV); vq->log = kmalloc_objs(*vq->log, dev->iov_limit); vq->heads = kmalloc_objs(*vq->heads, dev->iov_limit); vq->nheads = kmalloc_array(dev->iov_limit, sizeof(*vq->nheads), GFP_KERNEL); if (!vq->indirect || !vq->log || !vq->heads || !vq->nheads) goto err_nomem; } return 0;
err_nomem: for (; i >= 0; --i) vhost_vq_free_iovecs(dev->vqs[i]); return -ENOMEM;}
static void vhost_dev_free_iovecs(struct vhost_dev *dev){ int i;
for (i = 0; i < dev->nvqs; ++i) vhost_vq_free_iovecs(dev->vqs[i]);}
bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len){ struct vhost_dev *dev = vq->dev;
if ((dev->byte_weight && total_len >= dev->byte_weight) || pkts >= dev->weight) { vhost_poll_queue(&vq->poll); return true; }
return false;}EXPORT_SYMBOL_GPL(vhost_exceeds_weight);
static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, unsigned int num){ size_t event __maybe_unused = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return size_add(struct_size(vq->avail, ring, num), event);}
static size_t vhost_get_used_size(struct vhost_virtqueue *vq, unsigned int num){ size_t event __maybe_unused = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return size_add(struct_size(vq->used, ring, num), event);}
static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, unsigned int num){ return sizeof(*vq->desc) * num;}
void vhost_dev_init(struct vhost_dev *dev, struct vhost_virtqueue **vqs, int nvqs, int iov_limit, int weight, int byte_weight, bool use_worker, int (*msg_handler)(struct vhost_dev *dev, u32 asid, struct vhost_iotlb_msg *msg)){ struct vhost_virtqueue *vq; int i;
dev->vqs = vqs; dev->nvqs = nvqs; mutex_init(&dev->mutex); dev->log_ctx = NULL; dev->umem = NULL; dev->iotlb = NULL; dev->mm = NULL; dev->iov_limit = iov_limit; dev->weight = weight; dev->byte_weight = byte_weight; dev->use_worker = use_worker; dev->msg_handler = msg_handler; dev->fork_owner = fork_from_owner_default; init_waitqueue_head(&dev->wait); INIT_LIST_HEAD(&dev->read_list); INIT_LIST_HEAD(&dev->pending_list); spin_lock_init(&dev->iotlb_lock); xa_init_flags(&dev->worker_xa, XA_FLAGS_ALLOC);
for (i = 0; i < dev->nvqs; ++i) { vq = dev->vqs[i]; vq->log = NULL; vq->indirect = NULL; vq->heads = NULL; vq->nheads = NULL; vq->dev = dev; mutex_init(&vq->mutex); vhost_vq_reset(dev, vq); if (vq->handle_kick) vhost_poll_init(&vq->poll, vq->handle_kick, EPOLLIN, dev, vq); }}EXPORT_SYMBOL_GPL(vhost_dev_init);
/* Caller should have device mutex */long vhost_dev_check_owner(struct vhost_dev *dev){ /* Are you the owner? If not, I don't think you mean to do that */ return dev->mm == current->mm ? 0 : -EPERM;}EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
struct vhost_attach_cgroups_struct { struct vhost_work work; struct task_struct *owner; int ret;};
static void vhost_attach_cgroups_work(struct vhost_work *work){ struct vhost_attach_cgroups_struct *s;
s = container_of(work, struct vhost_attach_cgroups_struct, work); s->ret = cgroup_attach_task_all(s->owner, current);}
static int vhost_attach_task_to_cgroups(struct vhost_worker *worker){ struct vhost_attach_cgroups_struct attach; int saved_cnt;
attach.owner = current;
vhost_work_init(&attach.work, vhost_attach_cgroups_work); vhost_worker_queue(worker, &attach.work);
mutex_lock(&worker->mutex);
/* * Bypass attachment_cnt check in __vhost_worker_flush: * Temporarily change it to INT_MAX to bypass the check */ saved_cnt = worker->attachment_cnt; worker->attachment_cnt = INT_MAX; __vhost_worker_flush(worker); worker->attachment_cnt = saved_cnt;
mutex_unlock(&worker->mutex);
return attach.ret;}
/* Caller should have device mutex */bool vhost_dev_has_owner(struct vhost_dev *dev){ return dev->mm;}EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
static void vhost_attach_mm(struct vhost_dev *dev){ /* No owner, become one */ if (dev->use_worker) { dev->mm = get_task_mm(current); } else { /* vDPA device does not use worker thread, so there's * no need to hold the address space for mm. This helps * to avoid deadlock in the case of mmap() which may * hold the refcnt of the file and depends on release * method to remove vma. */ dev->mm = current->mm; mmgrab(dev->mm); }}
static void vhost_detach_mm(struct vhost_dev *dev){ if (!dev->mm) return;
if (dev->use_worker) mmput(dev->mm); else mmdrop(dev->mm);
dev->mm = NULL;}
static void vhost_worker_destroy(struct vhost_dev *dev, struct vhost_worker *worker){ if (!worker) return;
WARN_ON(!llist_empty(&worker->work_list)); xa_erase(&dev->worker_xa, worker->id); worker->ops->stop(worker); kfree(worker);}
static void vhost_workers_free(struct vhost_dev *dev, bool sync){ struct vhost_worker *worker; unsigned long i;
if (!dev->use_worker) return;
for (i = 0; i < dev->nvqs; i++) rcu_assign_pointer(dev->vqs[i]->worker, NULL);
/* * This path can be reached by lockless work queuers. In this case * vhost_vq_work_queue() reads vq->worker under rcu_read_lock(), so a * RCU reader that fetched a worker before we cleared the pointers above * may still be queueing work on it. Wait for those readers to finish, * then flush so any work they queued runs (clearing VHOST_WORK_QUEUED) * before the workers are freed. Notably, that is the case for the * vsock RESET_OWNER path. */ if (sync) { synchronize_rcu(); vhost_dev_flush(dev); }
/* * Free the default worker we created and cleanup workers userspace * created but couldn't clean up (it forgot or crashed). */ xa_for_each(&dev->worker_xa, i, worker) vhost_worker_destroy(dev, worker); xa_destroy(&dev->worker_xa);}
static void vhost_task_wakeup(struct vhost_worker *worker){ return vhost_task_wake(worker->vtsk);}
static void vhost_kthread_wakeup(struct vhost_worker *worker){ wake_up_process(worker->kthread_task);}
static void vhost_task_do_stop(struct vhost_worker *worker){ return vhost_task_stop(worker->vtsk);}
static void vhost_kthread_do_stop(struct vhost_worker *worker){ kthread_stop(worker->kthread_task);}
static int vhost_task_worker_create(struct vhost_worker *worker, struct vhost_dev *dev, const char *name){ struct vhost_task *vtsk; u32 id; int ret;
vtsk = vhost_task_create(vhost_run_work_list, vhost_worker_killed, worker, name); if (IS_ERR(vtsk)) return PTR_ERR(vtsk);
worker->vtsk = vtsk; vhost_task_start(vtsk); ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL); if (ret < 0) { vhost_task_do_stop(worker); return ret; } worker->id = id; return 0;}
static int vhost_kthread_worker_create(struct vhost_worker *worker, struct vhost_dev *dev, const char *name){ struct task_struct *task; u32 id; int ret;
task = kthread_create(vhost_run_work_kthread_list, worker, "%s", name); if (IS_ERR(task)) return PTR_ERR(task);
worker->kthread_task = task; wake_up_process(task); ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL); if (ret < 0) goto stop_worker;
ret = vhost_attach_task_to_cgroups(worker); if (ret) goto free_id;
worker->id = id; return 0;
free_id: xa_erase(&dev->worker_xa, id);stop_worker: vhost_kthread_do_stop(worker); return ret;}
static const struct vhost_worker_ops kthread_ops = { .create = vhost_kthread_worker_create, .stop = vhost_kthread_do_stop, .wakeup = vhost_kthread_wakeup,};
static const struct vhost_worker_ops vhost_task_ops = { .create = vhost_task_worker_create, .stop = vhost_task_do_stop, .wakeup = vhost_task_wakeup,};
static struct vhost_worker *vhost_worker_create(struct vhost_dev *dev){ struct vhost_worker *worker; char name[TASK_COMM_LEN]; int ret; const struct vhost_worker_ops *ops = dev->fork_owner ? &vhost_task_ops : &kthread_ops;
worker = kzalloc_obj(*worker, GFP_KERNEL_ACCOUNT); if (!worker) return NULL;
worker->dev = dev; worker->ops = ops; snprintf(name, sizeof(name), "vhost-%d", current->pid);
mutex_init(&worker->mutex); init_llist_head(&worker->work_list); worker->kcov_handle = kcov_common_handle(); ret = ops->create(worker, dev, name); if (ret < 0) goto free_worker;
return worker;
free_worker: kfree(worker); return NULL;}
/* Caller must have device mutex */static void __vhost_vq_attach_worker(struct vhost_virtqueue *vq, struct vhost_worker *worker){ struct vhost_worker *old_worker;
mutex_lock(&worker->mutex); if (worker->killed) { mutex_unlock(&worker->mutex); return; }
mutex_lock(&vq->mutex);
old_worker = rcu_dereference_check(vq->worker, lockdep_is_held(&vq->mutex)); rcu_assign_pointer(vq->worker, worker); worker->attachment_cnt++;
if (!old_worker) { mutex_unlock(&vq->mutex); mutex_unlock(&worker->mutex); return; } mutex_unlock(&vq->mutex); mutex_unlock(&worker->mutex);
/* * Take the worker mutex to make sure we see the work queued from * device wide flushes which doesn't use RCU for execution. */ mutex_lock(&old_worker->mutex); if (old_worker->killed) { mutex_unlock(&old_worker->mutex); return; }
/* * We don't want to call synchronize_rcu for every vq during setup * because it will slow down VM startup. If we haven't done * VHOST_SET_VRING_KICK and not done the driver specific * SET_ENDPOINT/RUNNING then we can skip the sync since there will * not be any works queued for scsi and net. */ mutex_lock(&vq->mutex); if (!vhost_vq_get_backend(vq) && !vq->kick) { mutex_unlock(&vq->mutex);
old_worker->attachment_cnt--; mutex_unlock(&old_worker->mutex); /* * vsock can queue anytime after VHOST_VSOCK_SET_GUEST_CID. * Warn if it adds support for multiple workers but forgets to * handle the early queueing case. */ WARN_ON(!old_worker->attachment_cnt && !llist_empty(&old_worker->work_list)); return; } mutex_unlock(&vq->mutex);
/* Make sure new vq queue/flush/poll calls see the new worker */ synchronize_rcu(); /* Make sure whatever was queued gets run */ __vhost_worker_flush(old_worker); old_worker->attachment_cnt--; mutex_unlock(&old_worker->mutex);}
/* Caller must have device mutex */static int vhost_vq_attach_worker(struct vhost_virtqueue *vq, struct vhost_vring_worker *info){ unsigned long index = info->worker_id; struct vhost_dev *dev = vq->dev; struct vhost_worker *worker;
if (!dev->use_worker) return -EINVAL;
worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT); if (!worker || worker->id != info->worker_id) return -ENODEV;
__vhost_vq_attach_worker(vq, worker); return 0;}
/* Caller must have device mutex */static int vhost_new_worker(struct vhost_dev *dev, struct vhost_worker_state *info){ struct vhost_worker *worker;
worker = vhost_worker_create(dev); if (!worker) return -ENOMEM;
info->worker_id = worker->id; return 0;}
/* Caller must have device mutex */static int vhost_free_worker(struct vhost_dev *dev, struct vhost_worker_state *info){ unsigned long index = info->worker_id; struct vhost_worker *worker;
worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT); if (!worker || worker->id != info->worker_id) return -ENODEV;
mutex_lock(&worker->mutex); if (worker->attachment_cnt || worker->killed) { mutex_unlock(&worker->mutex); return -EBUSY; } /* * A flush might have raced and snuck in before attachment_cnt was set * to zero. Make sure flushes are flushed from the queue before * freeing. */ __vhost_worker_flush(worker); mutex_unlock(&worker->mutex);
vhost_worker_destroy(dev, worker); return 0;}
static int vhost_get_vq_from_user(struct vhost_dev *dev, void __user *argp, struct vhost_virtqueue **vq, u32 *id){ u32 __user *idxp = argp; u32 idx; long r;
r = get_user(idx, idxp); if (r < 0) return r;
if (idx >= dev->nvqs) return -ENOBUFS;
idx = array_index_nospec(idx, dev->nvqs);
*vq = dev->vqs[idx]; *id = idx; return 0;}
/* Caller must have device mutex */long vhost_worker_ioctl(struct vhost_dev *dev, unsigned int ioctl, void __user *argp){ struct vhost_vring_worker ring_worker; struct vhost_worker_state state; struct vhost_worker *worker; struct vhost_virtqueue *vq; long ret; u32 idx;
if (!dev->use_worker) return -EINVAL;
if (!vhost_dev_has_owner(dev)) return -EINVAL;
ret = vhost_dev_check_owner(dev); if (ret) return ret;
switch (ioctl) { /* dev worker ioctls */ case VHOST_NEW_WORKER: /* * vhost_tasks will account for worker threads under the parent's * NPROC value but kthreads do not. To avoid userspace overflowing * the system with worker threads fork_owner must be true. */ if (!dev->fork_owner) return -EFAULT;
ret = vhost_new_worker(dev, &state); if (!ret && copy_to_user(argp, &state, sizeof(state))) ret = -EFAULT; return ret; case VHOST_FREE_WORKER: if (copy_from_user(&state, argp, sizeof(state))) return -EFAULT; return vhost_free_worker(dev, &state); /* vring worker ioctls */ case VHOST_ATTACH_VRING_WORKER: case VHOST_GET_VRING_WORKER: break; default: return -ENOIOCTLCMD; }
ret = vhost_get_vq_from_user(dev, argp, &vq, &idx); if (ret) return ret;
switch (ioctl) { case VHOST_ATTACH_VRING_WORKER: if (copy_from_user(&ring_worker, argp, sizeof(ring_worker))) { ret = -EFAULT; break; }
ret = vhost_vq_attach_worker(vq, &ring_worker); break; case VHOST_GET_VRING_WORKER: worker = rcu_dereference_check(vq->worker, lockdep_is_held(&dev->mutex)); if (!worker) { ret = -EINVAL; break; }
ring_worker.index = idx; ring_worker.worker_id = worker->id;
if (copy_to_user(argp, &ring_worker, sizeof(ring_worker))) ret = -EFAULT; break; default: ret = -ENOIOCTLCMD; break; }
return ret;}EXPORT_SYMBOL_GPL(vhost_worker_ioctl);
/* Caller should have device mutex */long vhost_dev_set_owner(struct vhost_dev *dev){ struct vhost_worker *worker; int err, i;
/* Is there an owner already? */ if (vhost_dev_has_owner(dev)) { err = -EBUSY; goto err_mm; }
vhost_attach_mm(dev);
err = vhost_dev_alloc_iovecs(dev); if (err) goto err_iovecs;
if (dev->use_worker) { /* * This should be done last, because vsock can queue work * before VHOST_SET_OWNER so it simplifies the failure path * below since we don't have to worry about vsock queueing * while we free the worker. */ worker = vhost_worker_create(dev); if (!worker) { err = -ENOMEM; goto err_worker; }
for (i = 0; i < dev->nvqs; i++) __vhost_vq_attach_worker(dev->vqs[i], worker); }
return 0;
err_worker: vhost_dev_free_iovecs(dev);err_iovecs: vhost_detach_mm(dev);err_mm: return err;}EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
static struct vhost_iotlb *iotlb_alloc(void){ if (max_iotlb_entries <= 0) return NULL;
return vhost_iotlb_alloc(max_iotlb_entries, VHOST_IOTLB_FLAG_RETIRE);}
struct vhost_iotlb *vhost_dev_reset_owner_prepare(void){ return iotlb_alloc();}EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
/* Caller should have device mutex */void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem, bool sync){ int i;
vhost_dev_cleanup(dev, sync);
dev->fork_owner = fork_from_owner_default; dev->umem = umem; /* We don't need VQ locks below since vhost_dev_cleanup makes sure * VQs aren't running. */ for (i = 0; i < dev->nvqs; ++i) dev->vqs[i]->umem = umem;}EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
void vhost_dev_stop(struct vhost_dev *dev){ int i;
for (i = 0; i < dev->nvqs; ++i) { if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) vhost_poll_stop(&dev->vqs[i]->poll); }
vhost_dev_flush(dev);}EXPORT_SYMBOL_GPL(vhost_dev_stop);
static void vhost_free_msg_locked(struct vhost_msg_node *node){ if (node->vq->iotlb_miss == node) node->vq->iotlb_miss = NULL; kfree(node);}
static void vhost_free_msg(struct vhost_dev *dev, struct vhost_msg_node *node){ spin_lock(&dev->iotlb_lock); vhost_free_msg_locked(node); spin_unlock(&dev->iotlb_lock);}
void vhost_clear_msg(struct vhost_dev *dev){ struct vhost_msg_node *node, *n;
spin_lock(&dev->iotlb_lock);
list_for_each_entry_safe(node, n, &dev->read_list, node) { list_del(&node->node); vhost_free_msg_locked(node); }
list_for_each_entry_safe(node, n, &dev->pending_list, node) { list_del(&node->node); vhost_free_msg_locked(node); }
spin_unlock(&dev->iotlb_lock);}EXPORT_SYMBOL_GPL(vhost_clear_msg);
void vhost_dev_cleanup(struct vhost_dev *dev, bool sync){ int i;
for (i = 0; i < dev->nvqs; ++i) { if (dev->vqs[i]->error_ctx) eventfd_ctx_put(dev->vqs[i]->error_ctx); if (dev->vqs[i]->kick) fput(dev->vqs[i]->kick); if (dev->vqs[i]->call_ctx.ctx) eventfd_ctx_put(dev->vqs[i]->call_ctx.ctx); vhost_vq_reset(dev, dev->vqs[i]); } vhost_dev_free_iovecs(dev); if (dev->log_ctx) eventfd_ctx_put(dev->log_ctx); dev->log_ctx = NULL; /* No one will access memory at this point */ vhost_iotlb_free(dev->umem); dev->umem = NULL; vhost_iotlb_free(dev->iotlb); dev->iotlb = NULL; vhost_clear_msg(dev); wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); vhost_workers_free(dev, sync); vhost_detach_mm(dev);}EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz){ u64 a = addr / VHOST_PAGE_SIZE / 8;
/* Make sure 64 bit math will not overflow. */ if (a > ULONG_MAX - (unsigned long)log_base || a + (unsigned long)log_base > ULONG_MAX) return false;
return access_ok(log_base + a, (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);}
/* Make sure 64 bit math will not overflow. */static bool vhost_overflow(u64 uaddr, u64 size){ if (uaddr > ULONG_MAX || size > ULONG_MAX) return true;
if (!size) return false;
return uaddr > ULONG_MAX - size + 1;}
/* Caller should have vq mutex and device mutex. */static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem, int log_all){ struct vhost_iotlb_map *map;
if (!umem) return false;
list_for_each_entry(map, &umem->list, link) { unsigned long a = map->addr;
if (vhost_overflow(map->addr, map->size)) return false;
if (!access_ok((void __user *)a, map->size)) return false; else if (log_all && !log_access_ok(log_base, map->start, map->size)) return false; } return true;}
static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, u64 addr, unsigned int size, int type){ const struct vhost_iotlb_map *map = vq->meta_iotlb[type];
if (!map) return NULL;
return (void __user *)(uintptr_t)(map->addr + addr - map->start);}
/* Can we switch to this memory table? *//* Caller should have device mutex but not vq mutex */static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem, int log_all){ int i;
for (i = 0; i < d->nvqs; ++i) { bool ok; bool log;
mutex_lock(&d->vqs[i]->mutex); log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL); /* If ring is inactive, will check when it's enabled. */ if (d->vqs[i]->private_data) ok = vq_memory_access_ok(d->vqs[i]->log_base, umem, log); else ok = true; mutex_unlock(&d->vqs[i]->mutex); if (!ok) return false; } return true;}
static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, struct iovec iov[], int iov_size, int access);
static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to, const void *from, unsigned size){ int ret;
if (!vq->iotlb) return __copy_to_user(to, from, size); else { /* This function should be called after iotlb * prefetch, which means we're sure that all vq * could be access through iotlb. So -EAGAIN should * not happen in this case. */ struct iov_iter t; void __user *uaddr = vhost_vq_meta_fetch(vq, (u64)(uintptr_t)to, size, VHOST_ADDR_USED);
if (uaddr) return __copy_to_user(uaddr, from, size);
ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, ARRAY_SIZE(vq->iotlb_iov), VHOST_ACCESS_WO); if (ret < 0) goto out; iov_iter_init(&t, ITER_DEST, vq->iotlb_iov, ret, size); ret = copy_to_iter(from, size, &t); if (ret == size) ret = 0; }out: return ret;}
static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, void __user *from, unsigned size){ int ret;
if (!vq->iotlb) return __copy_from_user(to, from, size); else { /* This function should be called after iotlb * prefetch, which means we're sure that vq * could be access through iotlb. So -EAGAIN should * not happen in this case. */ void __user *uaddr = vhost_vq_meta_fetch(vq, (u64)(uintptr_t)from, size, VHOST_ADDR_DESC); struct iov_iter f;
if (uaddr) return __copy_from_user(to, uaddr, size);
ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, ARRAY_SIZE(vq->iotlb_iov), VHOST_ACCESS_RO); if (ret < 0) { vq_err(vq, "IOTLB translation failure: uaddr " "%p size 0x%llx\n", from, (unsigned long long) size); goto out; } iov_iter_init(&f, ITER_SOURCE, vq->iotlb_iov, ret, size); ret = copy_from_iter(to, size, &f); if (ret == size) ret = 0; }
out: return ret;}
static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq, void __user *addr, unsigned int size, int type){ int ret;
ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, ARRAY_SIZE(vq->iotlb_iov), VHOST_ACCESS_RO); if (ret < 0) { vq_err(vq, "IOTLB translation failure: uaddr " "%p size 0x%llx\n", addr, (unsigned long long) size); return NULL; }
if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { vq_err(vq, "Non atomic userspace memory access: uaddr " "%p size 0x%llx\n", addr, (unsigned long long) size); return NULL; }
return vq->iotlb_iov[0].iov_base;}
/* This function should be called after iotlb * prefetch, which means we're sure that vq * could be access through iotlb. So -EAGAIN should * not happen in this case. */static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, void __user *addr, unsigned int size, int type){ void __user *uaddr = vhost_vq_meta_fetch(vq, (u64)(uintptr_t)addr, size, type); if (uaddr) return uaddr;
return __vhost_get_user_slow(vq, addr, size, type);}
#define vhost_put_user(vq, x, ptr) \({ \ int ret; \ if (!vq->iotlb) { \ ret = put_user(x, ptr); \ } else { \ __typeof__(ptr) to = \ (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ sizeof(*ptr), VHOST_ADDR_USED); \ if (to != NULL) \ ret = put_user(x, to); \ else \ ret = -EFAULT; \ } \ ret; \})
static inline int vhost_put_avail_event(struct vhost_virtqueue *vq){ return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), vhost_avail_event(vq));}
static inline int vhost_put_used(struct vhost_virtqueue *vq, struct vring_used_elem *head, int idx, int count){ return vhost_copy_to_user(vq, vq->used->ring + idx, head, count * sizeof(*head));}
static inline int vhost_put_used_flags(struct vhost_virtqueue *vq)
{ return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), &vq->used->flags);}
static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)
{ return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), &vq->used->idx);}
#define vhost_get_user(vq, x, ptr, type) \({ \ int ret; \ if (!vq->iotlb) { \ ret = get_user(x, ptr); \ } else { \ __typeof__(ptr) from = \ (__typeof__(ptr)) __vhost_get_user(vq, ptr, \ sizeof(*ptr), \ type); \ if (from != NULL) \ ret = get_user(x, from); \ else \ ret = -EFAULT; \ } \ ret; \})
#define vhost_get_avail(vq, x, ptr) \ vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
#define vhost_get_used(vq, x, ptr) \ vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
static void vhost_dev_lock_vqs(struct vhost_dev *d){ int i = 0; for (i = 0; i < d->nvqs; ++i) mutex_lock_nested(&d->vqs[i]->mutex, i);}
static void vhost_dev_unlock_vqs(struct vhost_dev *d){ int i = 0; for (i = 0; i < d->nvqs; ++i) mutex_unlock(&d->vqs[i]->mutex);}
static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq){ __virtio16 idx; u16 avail_idx; int r;
r = vhost_get_avail(vq, idx, &vq->avail->idx); if (unlikely(r < 0)) { vq_err(vq, "Failed to access available index at %p (%d)\n", &vq->avail->idx, r); return r; }
/* Check it isn't doing very strange thing with available indexes */ avail_idx = vhost16_to_cpu(vq, idx); if (unlikely((u16)(avail_idx - vq->last_avail_idx) > vq->num)) { vq_err(vq, "Invalid available index change from %u to %u", vq->last_avail_idx, avail_idx); return -EINVAL; }
/* We're done if there is nothing new */ if (avail_idx == vq->avail_idx) return 0;
vq->avail_idx = avail_idx;
/* * We updated vq->avail_idx so we need a memory barrier between * the index read above and the caller reading avail ring entries. */ smp_rmb(); return 1;}
static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, __virtio16 *head, int idx){ return vhost_get_avail(vq, *head, &vq->avail->ring[idx & (vq->num - 1)]);}
static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, __virtio16 *flags){ return vhost_get_avail(vq, *flags, &vq->avail->flags);}
static inline int vhost_get_used_event(struct vhost_virtqueue *vq, __virtio16 *event){ return vhost_get_avail(vq, *event, vhost_used_event(vq));}
static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, __virtio16 *idx){ return vhost_get_used(vq, *idx, &vq->used->idx);}
static inline int vhost_get_desc(struct vhost_virtqueue *vq, struct vring_desc *desc, int idx){ return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));}
static void vhost_iotlb_notify_vq(struct vhost_dev *d, struct vhost_iotlb_msg *msg){ struct vhost_msg_node *node, *n;
spin_lock(&d->iotlb_lock);
list_for_each_entry_safe(node, n, &d->pending_list, node) { struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb; if (msg->iova <= vq_msg->iova && msg->iova + msg->size - 1 >= vq_msg->iova && vq_msg->type == VHOST_IOTLB_MISS) { vhost_poll_queue(&node->vq->poll); list_del(&node->node); vhost_free_msg_locked(node); } }
spin_unlock(&d->iotlb_lock);}
static bool umem_access_ok(u64 uaddr, u64 size, int access){ unsigned long a = uaddr;
/* Make sure 64 bit math will not overflow. */ if (vhost_overflow(uaddr, size)) return false;
if ((access & VHOST_ACCESS_RO) && !access_ok((void __user *)a, size)) return false; if ((access & VHOST_ACCESS_WO) && !access_ok((void __user *)a, size)) return false; return true;}
static int vhost_process_iotlb_msg(struct vhost_dev *dev, u32 asid, struct vhost_iotlb_msg *msg){ int ret = 0;
if (asid != 0) return -EINVAL;
mutex_lock(&dev->mutex); vhost_dev_lock_vqs(dev); switch (msg->type) { case VHOST_IOTLB_UPDATE: if (!dev->iotlb) { ret = -EFAULT; break; } if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) { ret = -EFAULT; break; } vhost_vq_meta_reset(dev); if (vhost_iotlb_add_range(dev->iotlb, msg->iova, msg->iova + msg->size - 1, msg->uaddr, msg->perm)) { ret = -ENOMEM; break; } vhost_iotlb_notify_vq(dev, msg); break; case VHOST_IOTLB_INVALIDATE: if (!dev->iotlb) { ret = -EFAULT; break; } if (!msg->size) { ret = -EINVAL; break; } vhost_vq_meta_reset(dev); vhost_iotlb_del_range(dev->iotlb, msg->iova, msg->iova + msg->size - 1); break; default: ret = -EINVAL; break; }
vhost_dev_unlock_vqs(dev); mutex_unlock(&dev->mutex);
return ret;}ssize_t vhost_chr_write_iter(struct vhost_dev *dev, struct iov_iter *from){ struct vhost_iotlb_msg msg; size_t offset; int type, ret; u32 asid = 0;
ret = copy_from_iter(&type, sizeof(type), from); if (ret != sizeof(type)) { ret = -EINVAL; goto done; }
switch (type) { case VHOST_IOTLB_MSG: /* There maybe a hole after type for V1 message type, * so skip it here. */ offset = offsetof(struct vhost_msg, iotlb) - sizeof(int); break; case VHOST_IOTLB_MSG_V2: if (vhost_backend_has_feature(dev->vqs[0], VHOST_BACKEND_F_IOTLB_ASID)) { ret = copy_from_iter(&asid, sizeof(asid), from); if (ret != sizeof(asid)) { ret = -EINVAL; goto done; } offset = 0; } else offset = sizeof(__u32); break; default: ret = -EINVAL; goto done; }
iov_iter_advance(from, offset); ret = copy_from_iter(&msg, sizeof(msg), from); if (ret != sizeof(msg)) { ret = -EINVAL; goto done; }
if (msg.type == VHOST_IOTLB_UPDATE && msg.size == 0) { ret = -EINVAL; goto done; }
if (dev->msg_handler) ret = dev->msg_handler(dev, asid, &msg); else ret = vhost_process_iotlb_msg(dev, asid, &msg); if (ret) { ret = -EFAULT; goto done; }
ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) : sizeof(struct vhost_msg_v2);done: return ret;}EXPORT_SYMBOL(vhost_chr_write_iter);
__poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev, poll_table *wait){ __poll_t mask = 0;
poll_wait(file, &dev->wait, wait);
if (!list_empty(&dev->read_list)) mask |= EPOLLIN | EPOLLRDNORM;
return mask;}EXPORT_SYMBOL(vhost_chr_poll);
ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, int noblock){ DEFINE_WAIT(wait); struct vhost_msg_node *node; ssize_t ret = 0; unsigned size = sizeof(struct vhost_msg);
if (iov_iter_count(to) < size) return 0;
while (1) { if (!noblock) prepare_to_wait(&dev->wait, &wait, TASK_INTERRUPTIBLE);
node = vhost_dequeue_msg(dev, &dev->read_list); if (node) break; if (noblock) { ret = -EAGAIN; break; } if (signal_pending(current)) { ret = -ERESTARTSYS; break; } if (!dev->iotlb) { ret = -EBADFD; break; }
schedule(); }
if (!noblock) finish_wait(&dev->wait, &wait);
if (node) { struct vhost_iotlb_msg *msg; void *start = &node->msg;
switch (node->msg.type) { case VHOST_IOTLB_MSG: size = sizeof(node->msg); msg = &node->msg.iotlb; break; case VHOST_IOTLB_MSG_V2: size = sizeof(node->msg_v2); msg = &node->msg_v2.iotlb; break; default: BUG(); break; }
ret = copy_to_iter(start, size, to); if (ret != size || msg->type != VHOST_IOTLB_MISS) { vhost_free_msg(dev, node); return ret; } vhost_enqueue_msg(dev, &dev->pending_list, node); }
return ret;}EXPORT_SYMBOL_GPL(vhost_chr_read_iter);
static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access){ struct vhost_dev *dev = vq->dev; struct vhost_msg_node *node; struct vhost_iotlb_msg *msg; bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2);
node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG); if (!node) return -ENOMEM;
if (v2) { node->msg_v2.type = VHOST_IOTLB_MSG_V2; msg = &node->msg_v2.iotlb; } else { msg = &node->msg.iotlb; }
msg->type = VHOST_IOTLB_MISS; msg->iova = iova; msg->perm = access;
spin_lock(&dev->iotlb_lock); /* VQ processing stops at the first miss until userspace resolves it. */ if (vq->iotlb_miss) { spin_unlock(&dev->iotlb_lock); kfree(node); return 0; }
vq->iotlb_miss = node; list_add_tail(&node->node, &dev->read_list); spin_unlock(&dev->iotlb_lock);
wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
return 0;}
static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, vring_desc_t __user *desc, vring_avail_t __user *avail, vring_used_t __user *used)
{ /* If an IOTLB device is present, the vring addresses are * GIOVAs. Access validation occurs at prefetch time. */ if (vq->iotlb) return true;
return access_ok(desc, vhost_get_desc_size(vq, num)) && access_ok(avail, vhost_get_avail_size(vq, num)) && access_ok(used, vhost_get_used_size(vq, num));}
static void vhost_vq_meta_update(struct vhost_virtqueue *vq, const struct vhost_iotlb_map *map, int type){ int access = (type == VHOST_ADDR_USED) ? VHOST_ACCESS_WO : VHOST_ACCESS_RO;
if (likely(map->perm & access)) vq->meta_iotlb[type] = map;}
static bool iotlb_access_ok(struct vhost_virtqueue *vq, int access, u64 addr, u64 len, int type){ const struct vhost_iotlb_map *map; struct vhost_iotlb *umem = vq->iotlb; u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
if (vhost_vq_meta_fetch(vq, addr, len, type)) return true;
while (len > s) { map = vhost_iotlb_itree_first(umem, addr, last); if (map == NULL || map->start > addr) { vhost_iotlb_miss(vq, addr, access); return false; } else if (!(map->perm & access)) { /* Report the possible access violation by * request another translation from userspace. */ return false; }
size = map->size - addr + map->start;
if (orig_addr == addr && size >= len) vhost_vq_meta_update(vq, map, type);
s += size; addr += size; }
return true;}
int vq_meta_prefetch(struct vhost_virtqueue *vq){ unsigned int num = vq->num;
/* * vhost_vq_invalidate_access() clears all three addresses together. * A single zero address may be a valid GIOVA in IOTLB mode. */ if (!vq->desc && !vq->avail && !vq->used) return 0;
if (!vq->iotlb) return 1;
return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc, vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail, vhost_get_avail_size(vq, num), VHOST_ADDR_AVAIL) && iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used, vhost_get_used_size(vq, num), VHOST_ADDR_USED);}EXPORT_SYMBOL_GPL(vq_meta_prefetch);
/* Can we log writes? *//* Caller should have device mutex but not vq mutex */bool vhost_log_access_ok(struct vhost_dev *dev){ return memory_access_ok(dev, dev->umem, 1);}EXPORT_SYMBOL_GPL(vhost_log_access_ok);
static bool vq_log_used_access_ok(struct vhost_virtqueue *vq, void __user *log_base, bool log_used, u64 log_addr){ /* If an IOTLB device is present, log_addr is a GIOVA that * will never be logged by log_used(). */ if (vq->iotlb) return true;
return !log_used || log_access_ok(log_base, log_addr, vhost_get_used_size(vq, vq->num));}
/* Verify access for write logging. *//* Caller should have vq mutex and device mutex */static bool vq_log_access_ok(struct vhost_virtqueue *vq, void __user *log_base){ return vq_memory_access_ok(log_base, vq->umem, vhost_has_feature(vq, VHOST_F_LOG_ALL)) && vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr);}
/* Can we start vq? *//* Caller should have vq mutex and device mutex */bool vhost_vq_access_ok(struct vhost_virtqueue *vq){ if (!vq_log_access_ok(vq, vq->log_base)) return false;
return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);}EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m){ struct vhost_memory mem, *newmem; struct vhost_memory_region *region; struct vhost_iotlb *newumem, *oldumem; unsigned long size = offsetof(struct vhost_memory, regions); int i;
if (copy_from_user(&mem, m, size)) return -EFAULT; if (mem.padding) return -EOPNOTSUPP; if (mem.nregions > max_mem_regions) return -E2BIG; if (max_iotlb_entries <= 0) return -EINVAL; newmem = kvzalloc_flex(*newmem, regions, mem.nregions); if (!newmem) return -ENOMEM;
memcpy(newmem, &mem, size); if (copy_from_user(newmem->regions, m->regions, flex_array_size(newmem, regions, mem.nregions))) { kvfree(newmem); return -EFAULT; }
newumem = iotlb_alloc(); if (!newumem) { kvfree(newmem); return -ENOMEM; }
for (region = newmem->regions; region < newmem->regions + mem.nregions; region++) { if (vhost_iotlb_add_range(newumem, region->guest_phys_addr, region->guest_phys_addr + region->memory_size - 1, region->userspace_addr, VHOST_MAP_RW)) goto err; }
if (!memory_access_ok(d, newumem, 0)) goto err;
oldumem = d->umem; d->umem = newumem;
/* All memory accesses are done under some VQ mutex. */ for (i = 0; i < d->nvqs; ++i) { mutex_lock(&d->vqs[i]->mutex); d->vqs[i]->umem = newumem; mutex_unlock(&d->vqs[i]->mutex); }
kvfree(newmem); vhost_iotlb_free(oldumem); return 0;
err: vhost_iotlb_free(newumem); kvfree(newmem); return -EFAULT;}
static long vhost_vring_set_num(struct vhost_dev *d, struct vhost_virtqueue *vq, void __user *argp){ struct vhost_vring_state s;
/* Resizing ring with an active backend? * You don't want to do that. */ if (vq->private_data) return -EBUSY;
if (copy_from_user(&s, argp, sizeof s)) return -EFAULT;
if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) return -EINVAL; vq->num = s.num;
return 0;}
static long vhost_vring_set_addr(struct vhost_dev *d, struct vhost_virtqueue *vq, void __user *argp){ struct vhost_vring_addr a;
if (copy_from_user(&a, argp, sizeof a)) return -EFAULT; if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) return -EOPNOTSUPP;
/* For 32bit, verify that the top 32bits of the user data are set to zero. */ if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || (u64)(unsigned long)a.used_user_addr != a.used_user_addr || (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) return -EFAULT;
/* Make sure it's safe to cast pointers to vring types. */ BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) || (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) || (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) return -EINVAL;
/* We only verify access here if backend is configured. * If it is not, we don't as size might not have been setup. * We will verify when backend is configured. */ if (vq->private_data) { if (!vq_access_ok(vq, vq->num, (void __user *)(unsigned long)a.desc_user_addr, (void __user *)(unsigned long)a.avail_user_addr, (void __user *)(unsigned long)a.used_user_addr)) return -EINVAL;
/* Also validate log access for used ring if enabled. */ if (!vq_log_used_access_ok(vq, vq->log_base, a.flags & (0x1 << VHOST_VRING_F_LOG), a.log_guest_addr)) return -EINVAL; }
vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); vq->desc = (void __user *)(unsigned long)a.desc_user_addr; vq->avail = (void __user *)(unsigned long)a.avail_user_addr; vq->log_addr = a.log_guest_addr; vq->used = (void __user *)(unsigned long)a.used_user_addr;
return 0;}
static long vhost_vring_set_num_addr(struct vhost_dev *d, struct vhost_virtqueue *vq, unsigned int ioctl, void __user *argp){ long r;
mutex_lock(&vq->mutex);
switch (ioctl) { case VHOST_SET_VRING_NUM: r = vhost_vring_set_num(d, vq, argp); break; case VHOST_SET_VRING_ADDR: r = vhost_vring_set_addr(d, vq, argp); break; default: BUG(); }
/* * The metadata cache holds the IOTLB mapping that backed the previous * desc/avail/used addresses and vring size, both of which are being * replaced here. iotlb_access_ok() takes a cache hit as proof that the * region was validated, so the stale entries have to go. */ __vhost_vq_meta_reset(vq);
mutex_unlock(&vq->mutex);
return r;}long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp){ struct file *eventfp, *filep = NULL; bool pollstart = false, pollstop = false; struct eventfd_ctx *ctx = NULL; struct vhost_virtqueue *vq; struct vhost_vring_state s; struct vhost_vring_file f; u32 idx; long r;
r = vhost_get_vq_from_user(d, argp, &vq, &idx); if (r < 0) return r;
if (ioctl == VHOST_SET_VRING_NUM || ioctl == VHOST_SET_VRING_ADDR) { return vhost_vring_set_num_addr(d, vq, ioctl, argp); }
mutex_lock(&vq->mutex);
switch (ioctl) { case VHOST_SET_VRING_BASE: /* Moving base with an active backend? * You don't want to do that. */ if (vq->private_data) { r = -EBUSY; break; } if (copy_from_user(&s, argp, sizeof s)) { r = -EFAULT; break; } if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) { vq->next_avail_head = vq->last_avail_idx = s.num & 0xffff; vq->last_used_idx = (s.num >> 16) & 0xffff; } else { if (s.num > 0xffff) { r = -EINVAL; break; } vq->next_avail_head = vq->last_avail_idx = s.num; } /* Forget the cached index value. */ vq->avail_idx = vq->last_avail_idx; break; case VHOST_GET_VRING_BASE: s.index = idx; if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) s.num = (u32)vq->last_avail_idx | ((u32)vq->last_used_idx << 16); else s.num = vq->last_avail_idx; if (copy_to_user(argp, &s, sizeof s)) r = -EFAULT; break; case VHOST_SET_VRING_KICK: if (copy_from_user(&f, argp, sizeof f)) { r = -EFAULT; break; } eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd); if (IS_ERR(eventfp)) { r = PTR_ERR(eventfp); break; } if (eventfp != vq->kick) { pollstop = (filep = vq->kick) != NULL; pollstart = (vq->kick = eventfp) != NULL; } else filep = eventfp; break; case VHOST_SET_VRING_CALL: if (copy_from_user(&f, argp, sizeof f)) { r = -EFAULT; break; } ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); if (IS_ERR(ctx)) { r = PTR_ERR(ctx); break; }
swap(ctx, vq->call_ctx.ctx); break; case VHOST_SET_VRING_ERR: if (copy_from_user(&f, argp, sizeof f)) { r = -EFAULT; break; } ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd); if (IS_ERR(ctx)) { r = PTR_ERR(ctx); break; } swap(ctx, vq->error_ctx); break; case VHOST_SET_VRING_ENDIAN: r = vhost_set_vring_endian(vq, argp); break; case VHOST_GET_VRING_ENDIAN: r = vhost_get_vring_endian(vq, idx, argp); break; case VHOST_SET_VRING_BUSYLOOP_TIMEOUT: if (copy_from_user(&s, argp, sizeof(s))) { r = -EFAULT; break; } vq->busyloop_timeout = s.num; break; case VHOST_GET_VRING_BUSYLOOP_TIMEOUT: s.index = idx; s.num = vq->busyloop_timeout; if (copy_to_user(argp, &s, sizeof(s))) r = -EFAULT; break; default: r = -ENOIOCTLCMD; }
if (pollstop && vq->handle_kick) vhost_poll_stop(&vq->poll);
if (!IS_ERR_OR_NULL(ctx)) eventfd_ctx_put(ctx); if (filep) fput(filep);
if (pollstart && vq->handle_kick) r = vhost_poll_start(&vq->poll, vq->kick);
mutex_unlock(&vq->mutex);
if (pollstop && vq->handle_kick) vhost_dev_flush(vq->poll.dev); return r;}EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
/* Caller must hold the device mutex. */void vhost_clear_device_iotlb(struct vhost_dev *d){ struct vhost_iotlb *iotlb; int i;
iotlb = d->iotlb; if (!iotlb) return;
vhost_dev_lock_vqs(d);
/* * vhost_dev_lock_vqs() takes all VQ mutexes in index order. Drop the * device-wide view while they are held, then clear each per-VQ view * and its cached ring access before releasing the locks. Workers * cannot observe a mixed address-space state during this handoff. */ d->iotlb = NULL;
for (i = 0; i < d->nvqs; ++i) { struct vhost_virtqueue *vq = d->vqs[i];
vq->iotlb = NULL; vhost_vq_invalidate_access(vq); }
vhost_dev_unlock_vqs(d); vhost_clear_msg(d); vhost_iotlb_free(iotlb); wake_up_interruptible_poll(&d->wait, EPOLLIN | EPOLLRDNORM);}EXPORT_SYMBOL_GPL(vhost_clear_device_iotlb);
int vhost_init_device_iotlb(struct vhost_dev *d){ struct vhost_iotlb *niotlb, *oiotlb; int i;
if (max_iotlb_entries <= 0) return -EINVAL;
niotlb = iotlb_alloc(); if (!niotlb) return -ENOMEM;
oiotlb = d->iotlb; d->iotlb = niotlb;
for (i = 0; i < d->nvqs; ++i) { struct vhost_virtqueue *vq = d->vqs[i];
mutex_lock(&vq->mutex); vq->iotlb = niotlb; if (oiotlb) __vhost_vq_meta_reset(vq); else vhost_vq_invalidate_access(vq); mutex_unlock(&vq->mutex); }
vhost_iotlb_free(oiotlb);
return 0;}EXPORT_SYMBOL_GPL(vhost_init_device_iotlb);
/* Caller must have device mutex */long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp){ struct eventfd_ctx *ctx; u64 p; long r; int i, fd;
/* If you are not the owner, you can become one */ if (ioctl == VHOST_SET_OWNER) { r = vhost_dev_set_owner(d); goto done; }
#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL if (ioctl == VHOST_SET_FORK_FROM_OWNER) { /* Only allow modification before owner is set */ if (vhost_dev_has_owner(d)) { r = -EBUSY; goto done; } u8 fork_owner_val;
if (get_user(fork_owner_val, (u8 __user *)argp)) { r = -EFAULT; goto done; } if (fork_owner_val != VHOST_FORK_OWNER_TASK && fork_owner_val != VHOST_FORK_OWNER_KTHREAD) { r = -EINVAL; goto done; } d->fork_owner = !!fork_owner_val; r = 0; goto done; } if (ioctl == VHOST_GET_FORK_FROM_OWNER) { u8 fork_owner_val = d->fork_owner;
if (fork_owner_val != VHOST_FORK_OWNER_TASK && fork_owner_val != VHOST_FORK_OWNER_KTHREAD) { r = -EINVAL; goto done; } if (put_user(fork_owner_val, (u8 __user *)argp)) { r = -EFAULT; goto done; } r = 0; goto done; }#endif
/* You must be the owner to do anything else */ r = vhost_dev_check_owner(d); if (r) goto done;
switch (ioctl) { case VHOST_SET_MEM_TABLE: r = vhost_set_memory(d, argp); break; case VHOST_SET_LOG_BASE: if (copy_from_user(&p, argp, sizeof p)) { r = -EFAULT; break; } if ((u64)(unsigned long)p != p) { r = -EFAULT; break; } for (i = 0; i < d->nvqs; ++i) { struct vhost_virtqueue *vq; void __user *base = (void __user *)(unsigned long)p; vq = d->vqs[i]; mutex_lock(&vq->mutex); /* If ring is inactive, will check when it's enabled. */ if (vq->private_data && !vq_log_access_ok(vq, base)) r = -EFAULT; else vq->log_base = base; mutex_unlock(&vq->mutex); } break; case VHOST_SET_LOG_FD: r = get_user(fd, (int __user *)argp); if (r < 0) break; ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); if (IS_ERR(ctx)) { r = PTR_ERR(ctx); break; } swap(ctx, d->log_ctx); for (i = 0; i < d->nvqs; ++i) { mutex_lock(&d->vqs[i]->mutex); d->vqs[i]->log_ctx = d->log_ctx; mutex_unlock(&d->vqs[i]->mutex); } if (ctx) eventfd_ctx_put(ctx); break; default: r = -ENOIOCTLCMD; break; }done: return r;}EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
/* TODO: This is really inefficient. We need something like get_user() * (instruction directly accesses the data, with an exception table entry * returning -EFAULT). See Documentation/arch/x86/exception-tables.rst. */static int set_bit_to_user(int nr, void __user *addr){ unsigned long log = (unsigned long)addr; struct page *page; void *base; int bit = nr + (log % PAGE_SIZE) * 8; int r;
r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); if (r < 0) return r; BUG_ON(r != 1); base = kmap_atomic(page); set_bit(bit, base); kunmap_atomic(base); unpin_user_pages_dirty_lock(&page, 1, true); return 0;}
static int log_write(void __user *log_base, u64 write_address, u64 write_length){ u64 write_page = write_address / VHOST_PAGE_SIZE; int r;
if (!write_length) return 0; write_length += write_address % VHOST_PAGE_SIZE; for (;;) { u64 base = (u64)(unsigned long)log_base; u64 log = base + write_page / 8; int bit = write_page % 8; if ((u64)(unsigned long)log != log) return -EFAULT; r = set_bit_to_user(bit, (void __user *)(unsigned long)log); if (r < 0) return r; if (write_length <= VHOST_PAGE_SIZE) break; write_length -= VHOST_PAGE_SIZE; write_page += 1; } return r;}
static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len){ struct vhost_iotlb *umem = vq->umem; struct vhost_iotlb_map *u; u64 start, end, l, min; int r; bool hit = false;
while (len) { min = len; /* More than one GPAs can be mapped into a single HVA. So * iterate all possible umems here to be safe. */ list_for_each_entry(u, &umem->list, link) { if (u->addr > hva - 1 + len || u->addr - 1 + u->size < hva) continue; start = max(u->addr, hva); end = min(u->addr - 1 + u->size, hva - 1 + len); l = end - start + 1; r = log_write(vq->log_base, u->start + start - u->addr, l); if (r < 0) return r; hit = true; min = min(l, min); }
if (!hit) return -EFAULT;
len -= min; hva += min; }
return 0;}
static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len){ struct iovec *iov = vq->log_iov; int i, ret;
if (!vq->iotlb) return log_write(vq->log_base, vq->log_addr + used_offset, len);
ret = translate_desc(vq, (uintptr_t)vq->used + used_offset, len, iov, 64, VHOST_ACCESS_WO); if (ret < 0) return ret;
for (i = 0; i < ret; i++) { ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base, iov[i].iov_len); if (ret) return ret; }
return 0;}
/* * vhost_log_write() - Log in dirty page bitmap * @vq: vhost virtqueue. * @log: Array of dirty memory in GPA. * @log_num: Size of vhost_log arrary. * @len: The total length of memory buffer to log in the dirty bitmap. * Some drivers may only partially use pages shared via the last * vring descriptor (i.e. vhost-net RX buffer). * Use (len == U64_MAX) to indicate the driver would log all * pages of vring descriptors. * @iov: Array of dirty memory in HVA. * @count: Size of iovec array. */int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, unsigned int log_num, u64 len, struct iovec *iov, int count){ int i, r;
/* Make sure data written is seen before log. */ smp_wmb();
if (vq->iotlb) { for (i = 0; i < count; i++) { r = log_write_hva(vq, (uintptr_t)iov[i].iov_base, iov[i].iov_len); if (r < 0) return r; } return 0; }
for (i = 0; i < log_num; ++i) { u64 l = min(log[i].len, len); r = log_write(vq->log_base, log[i].addr, l); if (r < 0) return r;
if (len != U64_MAX) len -= l; }
if (vq->log_ctx) eventfd_signal(vq->log_ctx);
return 0;}EXPORT_SYMBOL_GPL(vhost_log_write);
static int vhost_update_used_flags(struct vhost_virtqueue *vq){ void __user *used; if (vhost_put_used_flags(vq)) return -EFAULT; if (unlikely(vq->log_used)) { /* Make sure the flag is seen before log. */ smp_wmb(); /* Log used flag write. */ used = &vq->used->flags; log_used(vq, (used - (void __user *)vq->used), sizeof vq->used->flags); if (vq->log_ctx) eventfd_signal(vq->log_ctx); } return 0;}
static int vhost_update_avail_event(struct vhost_virtqueue *vq){ if (vhost_put_avail_event(vq)) return -EFAULT; if (unlikely(vq->log_used)) { void __user *used; /* Make sure the event is seen before log. */ smp_wmb(); /* Log avail event write */ used = vhost_avail_event(vq); log_used(vq, (used - (void __user *)vq->used), sizeof *vhost_avail_event(vq)); if (vq->log_ctx) eventfd_signal(vq->log_ctx); } return 0;}
int vhost_vq_init_access(struct vhost_virtqueue *vq){ __virtio16 last_used_idx; int r; bool is_le = vq->is_le;
if (!vq->private_data) return 0;
vhost_init_is_le(vq);
r = vhost_update_used_flags(vq); if (r) goto err; vq->signalled_used_valid = false; if (!vq->iotlb && !access_ok(&vq->used->idx, sizeof vq->used->idx)) { r = -EFAULT; goto err; } r = vhost_get_used_idx(vq, &last_used_idx); if (r) { vq_err(vq, "Can't access used idx at %p\n", &vq->used->idx); goto err; } vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); return 0;
err: vq->is_le = is_le; return r;}EXPORT_SYMBOL_GPL(vhost_vq_init_access);
static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, struct iovec iov[], int iov_size, int access){ const struct vhost_iotlb_map *map; struct vhost_dev *dev = vq->dev; struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem; struct iovec *_iov; u64 s = 0, last = addr + len - 1; int ret = 0;
while ((u64)len > s) { u64 size; if (unlikely(ret >= iov_size)) { ret = -ENOBUFS; break; }
map = vhost_iotlb_itree_first(umem, addr, last); if (map == NULL || map->start > addr) { if (umem != dev->iotlb) { ret = -EFAULT; break; } ret = -EAGAIN; break; } else if (!(map->perm & access)) { ret = -EPERM; break; }
_iov = iov + ret; size = map->size - addr + map->start; _iov->iov_len = min((u64)len - s, size); _iov->iov_base = (void __user *)(unsigned long) (map->addr + addr - map->start); s += size; addr += size; ++ret; }
if (ret == -EAGAIN) vhost_iotlb_miss(vq, addr, access); return ret;}
/* Each buffer in the virtqueues is actually a chain of descriptors. This * function returns the next descriptor in the chain, * or -1U if we're at the end. */static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc){ unsigned int next;
/* If this descriptor says it doesn't chain, we're done. */ if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) return -1U;
/* Check they're not leading us off end of descriptors. */ next = vhost16_to_cpu(vq, READ_ONCE(desc->next)); return next;}
static int get_indirect(struct vhost_virtqueue *vq, struct iovec iov[], unsigned int iov_size, unsigned int *out_num, unsigned int *in_num, struct vhost_log *log, unsigned int *log_num, struct vring_desc *indirect){ struct vring_desc desc; unsigned int i = 0, count, found = 0; u32 len = vhost32_to_cpu(vq, indirect->len); struct iov_iter from; int ret, access;
/* Sanity check */ if (unlikely(len % sizeof desc)) { vq_err(vq, "Invalid length in indirect descriptor: " "len 0x%llx not multiple of 0x%zx\n", (unsigned long long)len, sizeof desc); return -EINVAL; }
ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, UIO_MAXIOV, VHOST_ACCESS_RO); if (unlikely(ret < 0)) { if (ret != -EAGAIN) vq_err(vq, "Translation failure %d in indirect.\n", ret); return ret; } iov_iter_init(&from, ITER_SOURCE, vq->indirect, ret, len); count = len / sizeof desc; /* Buffers are chained via a 16 bit next field, so * we can have at most 2^16 of these. */ if (unlikely(count > USHRT_MAX + 1)) { vq_err(vq, "Indirect buffer length too big: %d\n", indirect->len); return -E2BIG; }
do { unsigned iov_count = *in_num + *out_num; if (unlikely(++found > count)) { vq_err(vq, "Loop detected: last one at %u " "indirect size %u\n", i, count); return -EINVAL; } if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) { vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); return -EINVAL; } if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); return -EINVAL; }
if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) access = VHOST_ACCESS_WO; else access = VHOST_ACCESS_RO;
ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), vhost32_to_cpu(vq, desc.len), iov + iov_count, iov_size - iov_count, access); if (unlikely(ret < 0)) { if (ret != -EAGAIN) vq_err(vq, "Translation failure %d indirect idx %d\n", ret, i); return ret; } /* If this is an input descriptor, increment that count. */ if (access == VHOST_ACCESS_WO) { *in_num += ret; if (unlikely(log && ret)) { log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); log[*log_num].len = vhost32_to_cpu(vq, desc.len); ++*log_num; } } else { /* If it's an output descriptor, they're all supposed * to come before any input descriptors. */ if (unlikely(*in_num)) { vq_err(vq, "Indirect descriptor " "has out after in: idx %d\n", i); return -EINVAL; } *out_num += ret; } } while ((i = next_desc(vq, &desc)) != -1); return 0;}
/** * vhost_get_vq_desc_n - Fetch the next available descriptor chain and build iovecs * @vq: target virtqueue * @iov: array that receives the scatter/gather segments * @iov_size: capacity of @iov in elements * @out_num: the number of output segments * @in_num: the number of input segments * @log: optional array to record addr/len for each writable segment; NULL if unused * @log_num: optional output; number of entries written to @log when provided * @ndesc: optional output; number of descriptors consumed from the available ring * (useful for rollback via vhost_discard_vq_desc) * * Extracts one available descriptor chain from @vq and translates guest addresses * into host iovecs. * * On success, advances @vq->last_avail_idx by 1 and @vq->next_avail_head by the * number of descriptors consumed (also stored via @ndesc when non-NULL). * * Return: * - head index in [0, @vq->num) on success; * - @vq->num if no descriptor is currently available; * - negative errno on failure */int vhost_get_vq_desc_n(struct vhost_virtqueue *vq, struct iovec iov[], unsigned int iov_size, unsigned int *out_num, unsigned int *in_num, struct vhost_log *log, unsigned int *log_num, unsigned int *ndesc){ bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER); struct vring_desc desc; unsigned int i, head, found = 0; u16 last_avail_idx = vq->last_avail_idx; __virtio16 ring_head; int ret, access, c = 0;
if (vq->avail_idx == vq->last_avail_idx) { ret = vhost_get_avail_idx(vq); if (unlikely(ret < 0)) return ret;
if (!ret) return vq->num; }
if (in_order) head = vq->next_avail_head & (vq->num - 1); else { /* Grab the next descriptor number they're * advertising, and increment the index we've seen. */ if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) { vq_err(vq, "Failed to read head: idx %d address %p\n", last_avail_idx, &vq->avail->ring[last_avail_idx % vq->num]); return -EFAULT; } head = vhost16_to_cpu(vq, ring_head); }
/* If their number is silly, that's an error. */ if (unlikely(head >= vq->num)) { vq_err(vq, "Guest says index %u > %u is available", head, vq->num); return -EINVAL; }
/* When we start there are none of either input nor output. */ *out_num = *in_num = 0; if (unlikely(log)) *log_num = 0;
i = head; do { unsigned iov_count = *in_num + *out_num; if (unlikely(i >= vq->num)) { vq_err(vq, "Desc index is %u > %u, head = %u", i, vq->num, head); return -EINVAL; } if (unlikely(++found > vq->num)) { vq_err(vq, "Loop detected: last one at %u " "vq size %u head %u\n", i, vq->num, head); return -EINVAL; } ret = vhost_get_desc(vq, &desc, i); if (unlikely(ret)) { vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", i, vq->desc + i); return -EFAULT; } if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { ret = get_indirect(vq, iov, iov_size, out_num, in_num, log, log_num, &desc); if (unlikely(ret < 0)) { if (ret != -EAGAIN) vq_err(vq, "Failure detected " "in indirect descriptor at idx %d\n", i); return ret; } ++c; continue; }
if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) access = VHOST_ACCESS_WO; else access = VHOST_ACCESS_RO; ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), vhost32_to_cpu(vq, desc.len), iov + iov_count, iov_size - iov_count, access); if (unlikely(ret < 0)) { if (ret != -EAGAIN) vq_err(vq, "Translation failure %d descriptor idx %d\n", ret, i); return ret; } if (access == VHOST_ACCESS_WO) { /* If this is an input descriptor, * increment that count. */ *in_num += ret; if (unlikely(log && ret)) { log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); log[*log_num].len = vhost32_to_cpu(vq, desc.len); ++*log_num; } } else { /* If it's an output descriptor, they're all supposed * to come before any input descriptors. */ if (unlikely(*in_num)) { vq_err(vq, "Descriptor has out after in: " "idx %d\n", i); return -EINVAL; } *out_num += ret; } ++c; } while ((i = next_desc(vq, &desc)) != -1);
/* On success, increment avail index. */ vq->last_avail_idx++; vq->next_avail_head += c;
if (ndesc) *ndesc = c;
/* Assume notifications from guest are disabled at this point, * if they aren't we would need to update avail_event index. */ BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); return head;}EXPORT_SYMBOL_GPL(vhost_get_vq_desc_n);
/* This looks in the virtqueue and for the first available buffer, and converts * it to an iovec for convenient access. Since descriptors consist of some * number of output then some number of input descriptors, it's actually two * iovecs, but we pack them into one and note how many of each there were. * * This function returns the descriptor number found, or vq->num (which is * never a valid descriptor number) if none was found. A negative code is * returned on error. */int vhost_get_vq_desc(struct vhost_virtqueue *vq, struct iovec iov[], unsigned int iov_size, unsigned int *out_num, unsigned int *in_num, struct vhost_log *log, unsigned int *log_num){ return vhost_get_vq_desc_n(vq, iov, iov_size, out_num, in_num, log, log_num, NULL);}EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
/** * vhost_discard_vq_desc - Reverse the effect of vhost_get_vq_desc_n() * @vq: target virtqueue * @nbufs: number of buffers to roll back * @ndesc: number of descriptors to roll back * * Rewinds the internal consumer cursors after a failed attempt to use buffers * returned by vhost_get_vq_desc_n(). */void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int nbufs, unsigned int ndesc){ vq->next_avail_head -= ndesc; vq->last_avail_idx -= nbufs;}EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
/* After we've used one of their buffers, we tell them about it. We'll then * want to notify the guest, using eventfd. */int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len){ struct vring_used_elem heads = { cpu_to_vhost32(vq, head), cpu_to_vhost32(vq, len) }; u16 nheads = 1;
return vhost_add_used_n(vq, &heads, &nheads, 1);}EXPORT_SYMBOL_GPL(vhost_add_used);
static int __vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, unsigned count){ vring_used_elem_t __user *used; u16 old, new; int start;
start = vq->last_used_idx & (vq->num - 1); used = vq->used->ring + start; if (vhost_put_used(vq, heads, start, count)) { vq_err(vq, "Failed to write used"); return -EFAULT; } if (unlikely(vq->log_used)) { /* Make sure data is seen before log. */ smp_wmb(); /* Log used ring entry write. */ log_used(vq, ((void __user *)used - (void __user *)vq->used), count * sizeof *used); } old = vq->last_used_idx; new = (vq->last_used_idx += count); /* If the driver never bothers to signal in a very long while, * used index might wrap around. If that happens, invalidate * signalled_used index we stored. TODO: make sure driver * signals at least once in 2^16 and remove this. */ if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) vq->signalled_used_valid = false; return 0;}
static int vhost_add_used_n_ooo(struct vhost_virtqueue *vq, struct vring_used_elem *heads, unsigned count){ int start, n, r;
start = vq->last_used_idx & (vq->num - 1); n = vq->num - start; if (n < count) { r = __vhost_add_used_n(vq, heads, n); if (r < 0) return r; heads += n; count -= n; } return __vhost_add_used_n(vq, heads, count);}
static int vhost_add_used_n_in_order(struct vhost_virtqueue *vq, struct vring_used_elem *heads, const u16 *nheads, unsigned count){ vring_used_elem_t __user *used; u16 old, new = vq->last_used_idx; int start, i;
if (!nheads) return -EINVAL;
start = vq->last_used_idx & (vq->num - 1); used = vq->used->ring + start;
for (i = 0; i < count; i++) { if (vhost_put_used(vq, &heads[i], start, 1)) { vq_err(vq, "Failed to write used"); return -EFAULT; } start += nheads[i]; new += nheads[i]; if (start >= vq->num) start -= vq->num; }
if (unlikely(vq->log_used)) { /* Make sure data is seen before log. */ smp_wmb(); /* Log used ring entry write. */ log_used(vq, ((void __user *)used - (void __user *)vq->used), (vq->num - start) * sizeof *used); if (start + count > vq->num) log_used(vq, 0, (start + count - vq->num) * sizeof *used); }
old = vq->last_used_idx; vq->last_used_idx = new; /* If the driver never bothers to signal in a very long while, * used index might wrap around. If that happens, invalidate * signalled_used index we stored. TODO: make sure driver * signals at least once in 2^16 and remove this. */ if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) vq->signalled_used_valid = false; return 0;}
/* After we've used one of their buffers, we tell them about it. We'll then * want to notify the guest, using eventfd. */int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, u16 *nheads, unsigned count){ bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER); int r;
if (!in_order || !nheads) r = vhost_add_used_n_ooo(vq, heads, count); else r = vhost_add_used_n_in_order(vq, heads, nheads, count);
if (r < 0) return r;
/* Make sure buffer is written before we update index. */ smp_wmb(); if (vhost_put_used_idx(vq)) { vq_err(vq, "Failed to increment used idx"); return -EFAULT; } if (unlikely(vq->log_used)) { /* Make sure used idx is seen before log. */ smp_wmb(); /* Log used index update. */ log_used(vq, offsetof(struct vring_used, idx), sizeof vq->used->idx); if (vq->log_ctx) eventfd_signal(vq->log_ctx); } return r;}EXPORT_SYMBOL_GPL(vhost_add_used_n);
static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq){ __u16 old, new; __virtio16 event; bool v; /* Flush out used index updates. This is paired * with the barrier that the Guest executes when enabling * interrupts. */ smp_mb();
if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && unlikely(vq->avail_idx == vq->last_avail_idx)) return true;
if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { __virtio16 flags; if (vhost_get_avail_flags(vq, &flags)) { vq_err(vq, "Failed to get flags"); return true; } return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); } old = vq->signalled_used; v = vq->signalled_used_valid; new = vq->signalled_used = vq->last_used_idx; vq->signalled_used_valid = true;
if (unlikely(!v)) return true;
if (vhost_get_used_event(vq, &event)) { vq_err(vq, "Failed to get used event idx"); return true; } return vring_need_event(vhost16_to_cpu(vq, event), new, old);}
/* This actually signals the guest, using eventfd. */void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq){ /* Signal the Guest tell them we used something up. */ if (vq->call_ctx.ctx && vhost_notify(dev, vq)) eventfd_signal(vq->call_ctx.ctx);}EXPORT_SYMBOL_GPL(vhost_signal);
/* And here's the combo meal deal. Supersize me! */void vhost_add_used_and_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq, unsigned int head, int len){ vhost_add_used(vq, head, len); vhost_signal(dev, vq);}EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
/* multi-buffer version of vhost_add_used_and_signal */void vhost_add_used_and_signal_n(struct vhost_dev *dev, struct vhost_virtqueue *vq, struct vring_used_elem *heads, u16 *nheads, unsigned count){ vhost_add_used_n(vq, heads, nheads, count); vhost_signal(dev, vq);}EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
/* return true if we're sure that available ring is empty */bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq){ int r;
if (vq->avail_idx != vq->last_avail_idx) return false;
r = vhost_get_avail_idx(vq);
/* Note: we treat error as non-empty here */ return r == 0;}EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
/* OK, now we need to know about added descriptors. */bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq){ int r;
if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) return false; vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { r = vhost_update_used_flags(vq); if (r) { vq_err(vq, "Failed to enable notification at %p: %d\n", &vq->used->flags, r); return false; } } else { r = vhost_update_avail_event(vq); if (r) { vq_err(vq, "Failed to update avail event index at %p: %d\n", vhost_avail_event(vq), r); return false; } } /* They could have slipped one in as we were doing that: make * sure it's written, then check again. */ smp_mb();
r = vhost_get_avail_idx(vq); /* Note: we treat error as empty here */ if (unlikely(r < 0)) return false;
return r;}EXPORT_SYMBOL_GPL(vhost_enable_notify);
/* We don't need to be notified again. */void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq){ int r;
if (vq->used_flags & VRING_USED_F_NO_NOTIFY) return; vq->used_flags |= VRING_USED_F_NO_NOTIFY; if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { r = vhost_update_used_flags(vq); if (r) vq_err(vq, "Failed to disable notification at %p: %d\n", &vq->used->flags, r); }}EXPORT_SYMBOL_GPL(vhost_disable_notify);
/* Create a new message. */struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type){ /* Make sure all padding within the structure is initialized. */ struct vhost_msg_node *node = kzalloc_obj(*node); if (!node) return NULL;
node->vq = vq; node->msg.type = type; return node;}EXPORT_SYMBOL_GPL(vhost_new_msg);
void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head, struct vhost_msg_node *node){ spin_lock(&dev->iotlb_lock); list_add_tail(&node->node, head); spin_unlock(&dev->iotlb_lock);
wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);}EXPORT_SYMBOL_GPL(vhost_enqueue_msg);
struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev, struct list_head *head){ struct vhost_msg_node *node = NULL;
spin_lock(&dev->iotlb_lock); if (!list_empty(head)) { node = list_first_entry(head, struct vhost_msg_node, node); list_del(&node->node); } spin_unlock(&dev->iotlb_lock);
return node;}EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
void vhost_set_backend_features(struct vhost_dev *dev, u64 features){ struct vhost_virtqueue *vq; int i;
mutex_lock(&dev->mutex); for (i = 0; i < dev->nvqs; ++i) { vq = dev->vqs[i]; mutex_lock(&vq->mutex); vq->acked_backend_features = features; mutex_unlock(&vq->mutex); } mutex_unlock(&dev->mutex);}EXPORT_SYMBOL_GPL(vhost_set_backend_features);
MODULE_VERSION("0.0.1");MODULE_LICENSE("GPL v2");MODULE_AUTHOR("Michael S. Tsirkin");MODULE_DESCRIPTION("Host kernel accelerator for virtio");