Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832// SPDX-License-Identifier: GPL-2.0// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.// Copyright (c) 2018, Linaro Limited
#include <linux/completion.h>#include <linux/device.h>#include <linux/dma-buf.h>#include <linux/dma-mapping.h>#include <linux/dma-resv.h>#include <linux/idr.h>#include <linux/list.h>#include <linux/miscdevice.h>#include <linux/module.h>#include <linux/of_address.h>#include <linux/of.h>#include <linux/platform_device.h>#include <linux/sort.h>#include <linux/of_platform.h>#include <linux/rpmsg.h>#include <linux/scatterlist.h>#include <linux/slab.h>#include <linux/firmware/qcom/qcom_scm.h>#include <uapi/misc/fastrpc.h>#include <linux/of_reserved_mem.h>#include <linux/bitfield.h>#include <linux/bits.h>#include <linux/bitops.h>#include <linux/compiler.h>#include <linux/iopoll.h>
#define ADSP_DOMAIN_ID (0)#define MDSP_DOMAIN_ID (1)#define SDSP_DOMAIN_ID (2)#define CDSP_DOMAIN_ID (3)#define GDSP_DOMAIN_ID (4)#define FASTRPC_MAX_SESSIONS 14#define FASTRPC_MAX_VMIDS 16#define FASTRPC_ALIGN 128#define FASTRPC_MAX_FDLIST 16#define FASTRPC_MAX_CRCLIST 64#define FASTRPC_CTX_MAX (256)#define FASTRPC_INIT_HANDLE 1#define FASTRPC_DSP_UTILITIES_HANDLE 2/* * Maximum handle value for static handles. * Static handles are pre-defined, fixed numeric values statically assigned * in the IDL file or FastRPC framework. */#define FASTRPC_MAX_STATIC_HANDLE (20)#define FASTRPC_CTXID_MASK GENMASK(15, 8)/* Sequence number occupies bits 63:16 of the ctxid / message context */#define FASTRPC_CTXID_SEQ_SHIFT 16#define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16)
/* * Newer DSP firmware implements a PD (Protection Domain) notification * framework that sends PD state notifications upon request. The PD exit * notification is unconditionally sent by the DSP with this fixed sentinel * in the context field rather than the context of an outstanding invocation. * Since the fastrpc driver does not support the DSP PD notification framework, * this message must be dropped rather than matched against the context idr. */#define FASTRPC_DSP_PD_NOTIFY_CTX 0xABCDABCD
#define INIT_FILELEN_MAX (2 * 1024 * 1024)#define INIT_FILE_NAMELEN_MAX (128)#define FASTRPC_DEVICE_NAME "fastrpc"
/* Add memory to static PD pool, protection thru XPU */#define ADSP_MMAP_HEAP_ADDR 4/* MAP static DMA buffer on DSP User PD */#define ADSP_MMAP_DMA_BUFFER 6/* Add memory to userPD pool, for user heap */#define ADSP_MMAP_ADD_PAGES 0x1000/* Add memory to userPD pool, for LLC heap */#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
#define DSP_UNSUPPORTED_API (0x80000414)/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */#define FASTRPC_MAX_DSP_ATTRIBUTES (256)#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
/* Retrives number of input buffers from the scalars parameter */#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
/* Retrives number of output buffers from the scalars parameter */#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
/* Retrives number of input handles from the scalars parameter */#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
/* Retrives number of output handles from the scalars parameter */#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \ REMOTE_SCALARS_OUTBUFS(sc) + \ REMOTE_SCALARS_INHANDLES(sc)+ \ REMOTE_SCALARS_OUTHANDLES(sc))#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \ (((attr & 0x07) << 29) | \ ((method & 0x1f) << 24) | \ ((in & 0xff) << 16) | \ ((out & 0xff) << 8) | \ ((oin & 0x0f) << 4) | \ (oout & 0x0f))
#define FASTRPC_SCALARS(method, in, out) \ FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
#define FASTRPC_CREATE_PROCESS_NARGS 6#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3/* Remote Method id table */#define FASTRPC_RMID_INIT_ATTACH 0#define FASTRPC_RMID_INIT_RELEASE 1#define FASTRPC_RMID_INIT_MMAP 4#define FASTRPC_RMID_INIT_MUNMAP 5#define FASTRPC_RMID_INIT_CREATE 6#define FASTRPC_RMID_INIT_CREATE_ATTR 7#define FASTRPC_RMID_INIT_CREATE_STATIC 8#define FASTRPC_RMID_INIT_MEM_MAP 10#define FASTRPC_RMID_INIT_MEM_UNMAP 11
/* Protection Domain(PD) ids */#define ROOT_PD (0)#define USER_PD (1)#define SENSORS_PD (2)
#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
/* Poll response number from remote processor for call completion */#define FASTRPC_POLL_RESPONSE (0xdecaf)
/* Polling mode timeout limit */#define FASTRPC_POLL_MAX_TIMEOUT_US (10000)
struct fastrpc_phy_page { dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */};
struct fastrpc_invoke_buf { u32 num; /* number of contiguous regions */ u32 pgidx; /* index to start of contiguous region */};
struct fastrpc_remote_dmahandle { s32 fd; /* dma handle fd */ u32 offset; /* dma handle offset */ u32 len; /* dma handle length */};
struct fastrpc_remote_buf { u64 pv; /* buffer pointer */ u64 len; /* length of buffer */};
union fastrpc_remote_arg { struct fastrpc_remote_buf buf; struct fastrpc_remote_dmahandle dma;};
struct fastrpc_mmap_rsp_msg { u64 vaddr;};
struct fastrpc_mmap_req_msg { s32 client_id; u32 flags; u64 vaddr; s32 num;};
struct fastrpc_mem_map_req_msg { s32 client_id; s32 fd; s32 offset; u32 flags; u64 vaddrin; s32 num; s32 data_len;};
struct fastrpc_munmap_req_msg { s32 client_id; u64 vaddr; u64 size;};
struct fastrpc_mem_unmap_req_msg { s32 client_id; s32 fd; u64 vaddrin; u64 len;};
struct fastrpc_msg { int client_id; /* process client id */ int tid; /* thread id */ u64 ctx; /* invoke caller context */ u32 handle; /* handle to invoke */ u32 sc; /* scalars structure describing the data */ dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */};
struct fastrpc_invoke_rsp { u64 ctx; /* invoke caller context */ int retval; /* invoke return value */};
struct fastrpc_buf_overlap { u64 start; u64 end; int raix; u64 mstart; u64 mend; u64 offset;};
struct fastrpc_buf { struct fastrpc_user *fl; struct dma_buf *dmabuf; struct device *dev; void *virt; dma_addr_t dma_addr; u64 size; /* Lock for dma buf attachments */ struct mutex lock; struct list_head attachments; /* mmap support */ struct list_head node; /* list of user requested mmaps */ uintptr_t raddr;};
struct fastrpc_dma_buf_attachment { struct device *dev; struct sg_table sgt; struct list_head node;};
struct fastrpc_map { struct list_head node; struct fastrpc_user *fl; int fd; struct dma_buf *buf; struct sg_table *table; struct dma_buf_attachment *attach; dma_addr_t dma_addr; u64 size; void *va; u64 len; u64 raddr; u32 attr; struct kref refcount;};
struct fastrpc_invoke_ctx { int nscalars; int nbufs; int retval; int pid; int client_id; u32 sc; u64 *fdlist; u32 *crc; /* Poll memory that DSP updates */ u32 *poll_addr; u64 ctxid; u64 msg_sz; /* work done status flag */ bool is_work_done; /* process updates poll memory instead of glink response */ bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; struct work_struct put_work; struct fastrpc_msg msg; struct fastrpc_user *fl; union fastrpc_remote_arg *rpra; struct fastrpc_map **maps; struct fastrpc_buf *buf; struct fastrpc_invoke_args *args; struct fastrpc_buf_overlap *olaps; struct fastrpc_channel_ctx *cctx;};
struct fastrpc_session_ctx { struct device *dev; int sid; bool used; bool valid;};
struct fastrpc_soc_data { u32 sid_pos; u32 dma_addr_bits_cdsp; u32 dma_addr_bits_default; bool poll_mode_supported;};
struct fastrpc_channel_ctx { int domain_id; int sesscount; int vmcount; struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS]; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; struct idr ctx_idr; struct list_head users; struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; /* Flag if audio PD init mem was allocated */ bool audio_init_mem; /* Audio PD reserved remote heap region */ phys_addr_t remote_heap_addr; u64 remote_heap_size; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; bool poll_mode_supported; /* Per-channel sequence counter; incremented on every context allocation */ atomic_t ctx_seq; u64 dma_mask; const struct fastrpc_soc_data *soc_data;};
struct fastrpc_device { struct fastrpc_channel_ctx *cctx; struct miscdevice miscdev; bool secure;};
struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; struct list_head mmaps;
struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sctx; struct fastrpc_buf *init_mem;
int client_id; int pd; bool is_secure_dev; /* Flags poll mode state */ bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; /* Reference count */ struct kref refcount;};
/* Extract SMMU PA from consolidated IOVA */static inline dma_addr_t fastrpc_ipa_to_dma_addr(struct fastrpc_channel_ctx *cctx, dma_addr_t iova){ if (!cctx->soc_data->sid_pos) return 0; return iova & GENMASK_ULL(cctx->soc_data->sid_pos - 1, 0);}
/* * Prepare the consolidated iova to send to DSP by prepending the SID * to smmu PA at the appropriate position */static inline u64 fastrpc_sid_offset(struct fastrpc_channel_ctx *cctx, struct fastrpc_session_ctx *sctx){ return (u64)sctx->sid << cctx->soc_data->sid_pos;}
static void fastrpc_free_map(struct kref *ref){ struct fastrpc_map *map;
map = container_of(ref, struct fastrpc_map, refcount);
if (map->table) { if (map->attr & FASTRPC_ATTR_SECUREMAP) { struct qcom_scm_vmperm perm; int vmid = map->fl->cctx->vmperms[0].vmid; u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid); int err = 0;
perm.vmid = QCOM_SCM_VMID_HLOS; perm.perm = QCOM_SCM_PERM_RWX; err = qcom_scm_assign_mem(map->dma_addr, map->len, &src_perms, &perm, 1); if (err) { dev_err(map->fl->sctx->dev, "Failed to assign memory dma_addr %pad size 0x%llx err %d\n", &map->dma_addr, map->len, err); return; } } dma_buf_unmap_attachment_unlocked(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); }
if (map->fl) { spin_lock(&map->fl->lock); list_del(&map->node); spin_unlock(&map->fl->lock); map->fl = NULL; }
kfree(map);}
static void fastrpc_map_put(struct fastrpc_map *map){ if (map) kref_put(&map->refcount, fastrpc_free_map);}
static int fastrpc_map_get(struct fastrpc_map *map){ if (!map) return -ENOENT;
return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;}
static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, struct fastrpc_map **ppmap, bool take_ref){ struct fastrpc_map *map = NULL; struct dma_buf *buf; int ret = -ENOENT;
buf = dma_buf_get(fd); if (IS_ERR(buf)) return PTR_ERR(buf);
spin_lock(&fl->lock); list_for_each_entry(map, &fl->maps, node) { if (map->fd != fd || map->buf != buf) continue;
if (take_ref) { ret = fastrpc_map_get(map); if (ret) break; }
*ppmap = map; ret = 0; break; } spin_unlock(&fl->lock);
dma_buf_put(buf);
return ret;}
static void fastrpc_buf_free(struct fastrpc_buf *buf){ if (!buf) return;
dma_free_coherent(buf->dev, buf->size, buf->virt, fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr)); kfree(buf);}
static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ struct fastrpc_buf *buf;
buf = kzalloc_obj(*buf); if (!buf) return -ENOMEM;
INIT_LIST_HEAD(&buf->attachments); INIT_LIST_HEAD(&buf->node); mutex_init(&buf->lock);
buf->fl = fl; buf->virt = NULL; buf->dma_addr = 0; buf->size = size; buf->dev = dev; buf->raddr = 0;
buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr, GFP_KERNEL); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf); return -ENOMEM; }
*obuf = buf;
return 0;}
static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ int ret; struct fastrpc_buf *buf;
ret = __fastrpc_buf_alloc(fl, dev, size, obuf); if (ret) return ret;
buf = *obuf;
if (fl->sctx && fl->sctx->sid) buf->dma_addr += fastrpc_sid_offset(fl->cctx, fl->sctx);
return 0;}
static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev, u64 size, struct fastrpc_buf **obuf){ struct device *rdev = &fl->cctx->rpdev->dev;
return __fastrpc_buf_alloc(fl, rdev, size, obuf);}
static void fastrpc_channel_ctx_free(struct kref *ref){ struct fastrpc_channel_ctx *cctx;
cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
idr_destroy(&cctx->ctx_idr); kfree(cctx);}
static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx){ kref_get(&cctx->refcount);}
static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx){ kref_put(&cctx->refcount, fastrpc_channel_ctx_free);}
static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx);
static void fastrpc_user_free(struct kref *ref){ struct fastrpc_user *fl = container_of(ref, struct fastrpc_user, refcount); struct fastrpc_invoke_ctx *ctx, *n; struct fastrpc_map *map, *m; struct fastrpc_buf *buf, *b;
fastrpc_buf_free(fl->init_mem);
list_for_each_entry_safe(ctx, n, &fl->pending, node) { list_del(&ctx->node); fastrpc_context_put(ctx); }
list_for_each_entry_safe(map, m, &fl->maps, node) fastrpc_map_put(map);
list_for_each_entry_safe(buf, b, &fl->mmaps, node) { list_del(&buf->node); fastrpc_buf_free(buf); }
fastrpc_channel_ctx_put(fl->cctx); mutex_destroy(&fl->mutex); kfree(fl);}
static void fastrpc_user_get(struct fastrpc_user *fl){ kref_get(&fl->refcount);}
static void fastrpc_user_put(struct fastrpc_user *fl){ kref_put(&fl->refcount, fastrpc_user_free);}
static void fastrpc_context_free(struct kref *ref){ struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; struct fastrpc_user *fl; unsigned long flags; int i;
ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; fl = ctx->fl;
for (i = 0; i < ctx->nbufs; i++) fastrpc_map_put(ctx->maps[i]);
fastrpc_buf_free(ctx->buf);
spin_lock_irqsave(&cctx->lock, flags); idr_remove(&cctx->ctx_idr, FIELD_GET(FASTRPC_CTXID_MASK, ctx->ctxid)); spin_unlock_irqrestore(&cctx->lock, flags);
kfree(ctx->maps); kfree(ctx->olaps); kfree(ctx);
/* Release the reference taken in fastrpc_context_alloc() */ fastrpc_user_put(fl); fastrpc_channel_ctx_put(cctx);}
static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx){ kref_get(&ctx->refcount);}
static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx){ kref_put(&ctx->refcount, fastrpc_context_free);}
static void fastrpc_context_put_wq(struct work_struct *work){ struct fastrpc_invoke_ctx *ctx = container_of(work, struct fastrpc_invoke_ctx, put_work);
fastrpc_context_put(ctx);}
#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)static int olaps_cmp(const void *a, const void *b){ struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a; struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b; /* sort with lowest starting buffer first */ int st = CMP(pa->start, pb->start); /* sort with highest ending buffer first */ int ed = CMP(pb->end, pa->end);
return st == 0 ? ed : st;}
static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx){ u64 max_end = 0; int i;
for (i = 0; i < ctx->nbufs; ++i) { ctx->olaps[i].start = ctx->args[i].ptr; ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length; ctx->olaps[i].raix = i; }
sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
for (i = 0; i < ctx->nbufs; ++i) { /* Falling inside previous range */ if (ctx->olaps[i].start < max_end) { ctx->olaps[i].mstart = max_end; ctx->olaps[i].mend = ctx->olaps[i].end; ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
if (ctx->olaps[i].end > max_end) { max_end = ctx->olaps[i].end; } else { ctx->olaps[i].mend = 0; ctx->olaps[i].mstart = 0; }
} else { ctx->olaps[i].mend = ctx->olaps[i].end; ctx->olaps[i].mstart = ctx->olaps[i].start; ctx->olaps[i].offset = 0; max_end = ctx->olaps[i].end; } }}
static struct fastrpc_invoke_ctx *fastrpc_context_alloc( struct fastrpc_user *user, u32 kernel, u32 sc, struct fastrpc_invoke_args *args){ struct fastrpc_channel_ctx *cctx = user->cctx; struct fastrpc_invoke_ctx *ctx = NULL; unsigned long flags; int ret;
ctx = kzalloc_obj(*ctx); if (!ctx) return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&ctx->node); ctx->fl = user; ctx->nscalars = REMOTE_SCALARS_LENGTH(sc); ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) + REMOTE_SCALARS_OUTBUFS(sc);
if (ctx->nscalars) { ctx->maps = kzalloc_objs(*ctx->maps, ctx->nscalars); if (!ctx->maps) { kfree(ctx); return ERR_PTR(-ENOMEM); } ctx->olaps = kzalloc_objs(*ctx->olaps, ctx->nscalars); if (!ctx->olaps) { kfree(ctx->maps); kfree(ctx); return ERR_PTR(-ENOMEM); } ctx->args = args; fastrpc_get_buff_overlaps(ctx); }
/* Released in fastrpc_context_put() */ fastrpc_channel_ctx_get(cctx); /* Take a reference to user, released in fastrpc_context_free() */ fastrpc_user_get(user);
ctx->sc = sc; ctx->retval = -1; ctx->pid = current->pid; ctx->client_id = user->client_id; ctx->cctx = cctx; init_completion(&ctx->work); INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
spin_lock(&user->lock); list_add_tail(&ctx->node, &user->pending); spin_unlock(&user->lock);
spin_lock_irqsave(&cctx->lock, flags); ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1, FASTRPC_CTX_MAX, GFP_ATOMIC); if (ret < 0) { spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret) | FIELD_PREP(FASTRPC_CTXID_SEQ_MASK, (u64)atomic_inc_return(&cctx->ctx_seq)); spin_unlock_irqrestore(&cctx->lock, flags);
kref_init(&ctx->refcount);
return ctx;err_idr: spin_lock(&user->lock); list_del(&ctx->node); spin_unlock(&user->lock); fastrpc_user_put(user); fastrpc_channel_ctx_put(cctx); kfree(ctx->maps); kfree(ctx->olaps); kfree(ctx);
return ERR_PTR(ret);}
static struct sg_table *fastrpc_map_dma_buf(struct dma_buf_attachment *attachment, enum dma_data_direction dir){ struct fastrpc_dma_buf_attachment *a = attachment->priv; struct sg_table *table; int ret;
table = &a->sgt;
ret = dma_map_sgtable(attachment->dev, table, dir, 0); if (ret) table = ERR_PTR(ret); return table;}
static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach, struct sg_table *table, enum dma_data_direction dir){ dma_unmap_sgtable(attach->dev, table, dir, 0);}
static void fastrpc_release(struct dma_buf *dmabuf){ struct fastrpc_buf *buffer = dmabuf->priv;
fastrpc_buf_free(buffer);}
static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment){ struct fastrpc_dma_buf_attachment *a; struct fastrpc_buf *buffer = dmabuf->priv; int ret;
a = kzalloc_obj(*a); if (!a) return -ENOMEM;
ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt, fastrpc_ipa_to_dma_addr(buffer->fl->cctx, buffer->dma_addr), buffer->size); if (ret < 0) { dev_err(buffer->dev, "failed to get scatterlist from DMA API\n"); kfree(a); return -EINVAL; }
a->dev = attachment->dev; INIT_LIST_HEAD(&a->node); attachment->priv = a;
mutex_lock(&buffer->lock); list_add(&a->node, &buffer->attachments); mutex_unlock(&buffer->lock);
return 0;}
static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment){ struct fastrpc_dma_buf_attachment *a = attachment->priv; struct fastrpc_buf *buffer = dmabuf->priv;
mutex_lock(&buffer->lock); list_del(&a->node); mutex_unlock(&buffer->lock); sg_free_table(&a->sgt); kfree(a);}
static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map){ struct fastrpc_buf *buf = dmabuf->priv;
iosys_map_set_vaddr(map, buf->virt);
return 0;}
static int fastrpc_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma){ struct fastrpc_buf *buf = dmabuf->priv; size_t size = vma->vm_end - vma->vm_start;
dma_resv_assert_held(dmabuf->resv);
return dma_mmap_coherent(buf->dev, vma, buf->virt, fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr), size);}
static const struct dma_buf_ops fastrpc_dma_buf_ops = { .attach = fastrpc_dma_buf_attach, .detach = fastrpc_dma_buf_detatch, .map_dma_buf = fastrpc_map_dma_buf, .unmap_dma_buf = fastrpc_unmap_dma_buf, .mmap = fastrpc_mmap, .vmap = fastrpc_vmap, .release = fastrpc_release,};
static dma_addr_t fastrpc_compute_dma_addr(struct fastrpc_user *fl, dma_addr_t sg_dma_addr){ return sg_dma_addr + fastrpc_sid_offset(fl->cctx, fl->sctx);}
static int fastrpc_map_attach(struct fastrpc_user *fl, int fd, u64 len, u32 attr, struct fastrpc_map **ppmap){ struct fastrpc_session_ctx *sess = fl->sctx; struct fastrpc_map *map = NULL; struct sg_table *table; struct scatterlist *sgl = NULL; int err = 0, sgl_index = 0;
map = kzalloc_obj(*map); if (!map) return -ENOMEM;
INIT_LIST_HEAD(&map->node); kref_init(&map->refcount);
map->fl = fl; map->fd = fd; map->buf = dma_buf_get(fd); if (IS_ERR(map->buf)) { err = PTR_ERR(map->buf); goto get_err; }
map->attach = dma_buf_attach(map->buf, sess->dev); if (IS_ERR(map->attach)) { dev_err(sess->dev, "Failed to attach dmabuf\n"); err = PTR_ERR(map->attach); goto attach_err; }
table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL); if (IS_ERR(table)) { err = PTR_ERR(table); goto map_err; } map->table = table;
if (attr & FASTRPC_ATTR_SECUREMAP) map->dma_addr = sg_phys(map->table->sgl); else map->dma_addr = fastrpc_compute_dma_addr(fl, sg_dma_address(map->table->sgl)); for_each_sg(map->table->sgl, sgl, map->table->nents, sgl_index) map->size += sg_dma_len(sgl); if (len > map->size) { dev_dbg(sess->dev, "Bad size passed len 0x%llx map size 0x%llx\n", len, map->size); err = -EINVAL; goto get_err; } map->va = sg_virt(map->table->sgl); map->len = len;
if (attr & FASTRPC_ATTR_SECUREMAP) { /* * If subsystem VMIDs are defined in DTSI, then do * hyp_assign from HLOS to those VM(s) */ u64 src_perms = BIT(QCOM_SCM_VMID_HLOS); struct qcom_scm_vmperm dst_perms[2] = {0};
dst_perms[0].vmid = QCOM_SCM_VMID_HLOS; dst_perms[0].perm = QCOM_SCM_PERM_RW; dst_perms[1].vmid = fl->cctx->vmperms[0].vmid; dst_perms[1].perm = QCOM_SCM_PERM_RWX; map->attr = attr; err = qcom_scm_assign_mem(map->dma_addr, (u64)map->len, &src_perms, dst_perms, 2); if (err) { dev_err(sess->dev, "Failed to assign memory with dma_addr %pad size 0x%llx err %d\n", &map->dma_addr, map->len, err); goto get_err; } } spin_lock(&fl->lock); list_add_tail(&map->node, &fl->maps); spin_unlock(&fl->lock); *ppmap = map;
return 0;
map_err: dma_buf_detach(map->buf, map->attach);attach_err: dma_buf_put(map->buf);get_err: fastrpc_map_put(map);
return err;}
static int fastrpc_map_create(struct fastrpc_user *fl, int fd, u64 len, u32 attr, struct fastrpc_map **ppmap){ if (!fastrpc_map_lookup(fl, fd, ppmap, true)) return 0;
return fastrpc_map_attach(fl, fd, len, attr, ppmap);}
/* * Fastrpc payload buffer with metadata looks like: * * >>>>>> START of METADATA <<<<<<<<< * +---------------------------------+ * | Arguments | * | type:(union fastrpc_remote_arg)| * | (0 - N) | * +---------------------------------+ * | Invoke Buffer list | * | type:(struct fastrpc_invoke_buf)| * | (0 - N) | * +---------------------------------+ * | Page info list | * | type:(struct fastrpc_phy_page) | * | (0 - N) | * +---------------------------------+ * | Optional info | * |(can be specific to SoC/Firmware)| * +---------------------------------+ * >>>>>>>> END of METADATA <<<<<<<<< * +---------------------------------+ * | Inline ARGS | * | (0-N) | * +---------------------------------+ */
static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx){ int size = 0;
size = (sizeof(struct fastrpc_remote_buf) + sizeof(struct fastrpc_invoke_buf) + sizeof(struct fastrpc_phy_page)) * ctx->nscalars + sizeof(u64) * FASTRPC_MAX_FDLIST + sizeof(u32) * FASTRPC_MAX_CRCLIST + sizeof(u32);
return size;}
static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen){ u64 size = 0; int oix;
size = ALIGN(metalen, FASTRPC_ALIGN); for (oix = 0; oix < ctx->nbufs; oix++) { int i = ctx->olaps[oix].raix;
if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
if (ctx->olaps[oix].offset == 0) size = ALIGN(size, FASTRPC_ALIGN);
size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart); } }
return size;}
static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx){ struct device *dev = ctx->fl->sctx->dev; int i, err;
for (i = 0; i < ctx->nscalars; ++i) {
if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 || ctx->args[i].length == 0) continue;
if (i < ctx->nbufs) err = fastrpc_map_create(ctx->fl, ctx->args[i].fd, ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); else err = fastrpc_map_attach(ctx->fl, ctx->args[i].fd, ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); if (err) { dev_err(dev, "Error Creating map %d\n", err); return -EINVAL; }
} return 0;}
static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len){ return (struct fastrpc_invoke_buf *)(&pra[len]);}
static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len){ return (struct fastrpc_phy_page *)(&buf[len]);}
static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx){ struct device *dev = ctx->fl->sctx->dev; union fastrpc_remote_arg *rpra; struct fastrpc_invoke_buf *list; struct fastrpc_phy_page *pages; int inbufs, i, oix, err = 0; u64 len, rlen, pkt_size; u64 pg_start, pg_end; uintptr_t args; int metalen;
inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); metalen = fastrpc_get_meta_size(ctx); pkt_size = fastrpc_get_payload_size(ctx, metalen);
err = fastrpc_create_maps(ctx); if (err) return err;
ctx->msg_sz = pkt_size;
if (ctx->fl->sctx->sid) err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf); else err = fastrpc_remote_heap_alloc(ctx->fl, dev, pkt_size, &ctx->buf); if (err) return err;
memset(ctx->buf->virt, 0, pkt_size); rpra = ctx->buf->virt; list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); pages = fastrpc_phy_page_start(list, ctx->nscalars); ctx->fdlist = (u64 *)(pages + ctx->nscalars); ctx->poll_addr = (u32 *)((uintptr_t)ctx->fdlist + sizeof(u64) * FASTRPC_MAX_FDLIST + sizeof(u32) * FASTRPC_MAX_CRCLIST);
args = (uintptr_t)ctx->buf->virt + metalen; rlen = pkt_size - metalen; ctx->rpra = rpra;
for (oix = 0; oix < ctx->nbufs; ++oix) { int mlen;
i = ctx->olaps[oix].raix; len = ctx->args[i].length;
rpra[i].buf.pv = 0; rpra[i].buf.len = len; list[i].num = len ? 1 : 0; list[i].pgidx = i;
if (!len) continue;
if (ctx->maps[i]) { struct vm_area_struct *vma = NULL;
rpra[i].buf.pv = (u64) ctx->args[i].ptr; pages[i].addr = ctx->maps[i]->dma_addr;
mmap_read_lock(current->mm); vma = vma_lookup(current->mm, ctx->args[i].ptr); if (vma) pages[i].addr += (ctx->args[i].ptr & PAGE_MASK) - vma->vm_start; mmap_read_unlock(current->mm);
pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT; pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >> PAGE_SHIFT; pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
} else {
if (ctx->olaps[oix].offset == 0) { rlen -= ALIGN(args, FASTRPC_ALIGN) - args; args = ALIGN(args, FASTRPC_ALIGN); }
mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
if (rlen < mlen) goto bail;
rpra[i].buf.pv = args - ctx->olaps[oix].offset; pages[i].addr = ctx->buf->dma_addr - ctx->olaps[oix].offset + (pkt_size - rlen); pages[i].addr = pages[i].addr & PAGE_MASK;
pg_start = (rpra[i].buf.pv & PAGE_MASK) >> PAGE_SHIFT; pg_end = ((rpra[i].buf.pv + len - 1) & PAGE_MASK) >> PAGE_SHIFT; pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE; args = args + mlen; rlen -= mlen; }
if (i < inbufs && !ctx->maps[i]) { void *dst = (void *)(uintptr_t)rpra[i].buf.pv; void *src = (void *)(uintptr_t)ctx->args[i].ptr;
if (!kernel) { if (copy_from_user(dst, (void __user *)src, len)) { err = -EFAULT; goto bail; } } else { memcpy(dst, src, len); } } }
for (i = ctx->nbufs; i < ctx->nscalars; ++i) { list[i].num = ctx->args[i].length ? 1 : 0; list[i].pgidx = i; if (ctx->maps[i]) { pages[i].addr = ctx->maps[i]->dma_addr; pages[i].size = ctx->maps[i]->size; } rpra[i].dma.fd = ctx->args[i].fd; rpra[i].dma.len = ctx->args[i].length; rpra[i].dma.offset = (u64) ctx->args[i].ptr; }
bail: if (err) dev_err(dev, "Error: get invoke args failed:%d\n", err);
return err;}
static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, u32 kernel){ union fastrpc_remote_arg *rpra = ctx->rpra; struct fastrpc_user *fl = ctx->fl; struct fastrpc_map *mmap = NULL; u64 *fdlist = ctx->fdlist; int i, inbufs; int ret = 0;
inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
for (i = inbufs; i < ctx->nbufs; ++i) { if (!ctx->maps[i]) { void *src = (void *)(uintptr_t)rpra[i].buf.pv; void *dst = (void *)(uintptr_t)ctx->args[i].ptr; u64 len = rpra[i].buf.len;
if (!kernel) { if (copy_to_user((void __user *)dst, src, len)) { ret = -EFAULT; goto cleanup_fdlist; } } else { memcpy(dst, src, len); } } }
cleanup_fdlist: /* Clean up fdlist which is updated by DSP */ for (i = 0; i < FASTRPC_MAX_FDLIST; i++) { if (!fdlist[i]) break; if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false)) fastrpc_map_put(mmap); }
return ret;}
static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, struct fastrpc_invoke_ctx *ctx, u32 kernel, uint32_t handle){ struct fastrpc_channel_ctx *cctx; struct fastrpc_user *fl = ctx->fl; struct fastrpc_msg *msg = &ctx->msg; int ret;
cctx = fl->cctx; msg->client_id = fl->client_id; msg->tid = current->pid;
if (kernel) msg->client_id = 0;
msg->ctx = ctx->ctxid | fl->pd; msg->handle = handle; msg->sc = ctx->sc; msg->addr = ctx->buf ? ctx->buf->dma_addr : 0; msg->size = roundup(ctx->msg_sz, PAGE_SIZE); fastrpc_context_get(ctx);
ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
if (ret) fastrpc_context_put(ctx);
return ret;
}
static u32 fastrpc_read_poll_addr(struct fastrpc_invoke_ctx *ctx){ dma_rmb(); return READ_ONCE(*ctx->poll_addr);}
static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx){ u32 val; int ret;
/* * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll_addr * or until another path marks the work done. */ ret = read_poll_timeout_atomic(fastrpc_read_poll_addr, val, (val == FASTRPC_POLL_RESPONSE) || ctx->is_work_done, 1, FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx);
if (!ret && val == FASTRPC_POLL_RESPONSE) { /* * DSP writes FASTRPC_POLL_RESPONSE to signal successful * completion via the poll path. */ ctx->is_work_done = true; ctx->retval = 0; }
if (ret == -ETIMEDOUT) ret = -EIO;
return ret;}
static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx, u32 kernel){ int err = 0;
if (kernel) { if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) err = -ETIMEDOUT; } else { err = wait_for_completion_interruptible(&ctx->work); }
return err;}
static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx, u32 kernel){ int err;
if (ctx->is_polled) { err = poll_for_remote_response(ctx); if (!err) return 0; /* If polling timed out or failed, move to normal response mode */ ctx->is_polled = false; }
return fastrpc_wait_for_response(ctx, kernel);}
static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, u32 handle, u32 sc, struct fastrpc_invoke_args *args){ struct fastrpc_invoke_ctx *ctx = NULL; struct fastrpc_buf *buf, *b;
int err = 0;
if (!fl->sctx) return -EINVAL;
if (!fl->cctx->rpdev) return -EPIPE;
if (handle == FASTRPC_INIT_HANDLE && !kernel) { dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle); return -EPERM; }
ctx = fastrpc_context_alloc(fl, kernel, sc, args); if (IS_ERR(ctx)) return PTR_ERR(ctx);
err = fastrpc_get_args(kernel, ctx); if (err) goto bail;
/* make sure that all CPU memory writes are seen by DSP */ dma_wmb(); /* Send invoke buffer to remote dsp */ err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle); if (err) goto bail;
/* * Set message context as polled if the call is for a user PD * dynamic module and user has enabled poll mode. */ if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && fl->poll_mode) ctx->is_polled = true;
err = fastrpc_wait_for_completion(ctx, kernel); if (err) goto bail;
/* make sure that all memory writes by DSP are seen by CPU */ dma_rmb(); /* populate all the output buffers with results */ err = fastrpc_put_args(ctx, kernel); if (err) goto bail;
/* Check the response from remote dsp */ err = ctx->retval; if (err) goto bail;
bail: if (err != -ERESTARTSYS && err != -ETIMEDOUT) { /* We are done with this compute context */ spin_lock(&fl->lock); list_del(&ctx->node); spin_unlock(&fl->lock); fastrpc_context_put(ctx); }
if (err == -ERESTARTSYS) { spin_lock(&fl->lock); list_for_each_entry_safe(buf, b, &fl->mmaps, node) { list_del(&buf->node); list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps); } spin_unlock(&fl->lock); }
if (err) dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
return err;}
static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request){ /* Check if the device node is non-secure and channel is secure*/ if (!fl->is_secure_dev && fl->cctx->secure) { /* * Allow untrusted applications to offload only to Unsigned PD when * channel is configured as secure and block untrusted apps on channel * that does not support unsigned PD offload */ if (!fl->cctx->unsigned_support || !unsigned_pd_request) { dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n"); return true; } }
return false;}
static int fastrpc_init_create_static_process(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_init_create_static init; struct fastrpc_invoke_args *args; struct fastrpc_phy_page pages[1]; struct fastrpc_channel_ctx *cctx = fl->cctx; char *name; int err; struct { int client_id; u32 namelen; u32 pageslen; } inbuf; u32 sc; unsigned long flags; bool sent_heap = false;
if (!cctx->remote_heap_addr || !cctx->remote_heap_size) { err = -ENOMEM; dev_err(fl->sctx->dev, "remote heap memory region is not added\n"); return err; }
args = kzalloc_objs(*args, FASTRPC_CREATE_STATIC_PROCESS_NARGS); if (!args) return -ENOMEM;
if (copy_from_user(&init, argp, sizeof(init))) { err = -EFAULT; goto err; }
if (init.namelen > INIT_FILE_NAMELEN_MAX) { err = -EINVAL; goto err; }
name = memdup_user(u64_to_user_ptr(init.name), init.namelen); if (IS_ERR(name)) { err = PTR_ERR(name); goto err; } inbuf.client_id = fl->client_id; inbuf.namelen = init.namelen; inbuf.pageslen = 0;
fl->pd = USER_PD;
args[0].ptr = (u64)(uintptr_t)&inbuf; args[0].length = sizeof(inbuf); args[0].fd = -1;
args[1].ptr = (u64)(uintptr_t)name; args[1].length = inbuf.namelen; args[1].fd = -1;
/* * Audio PD is a static PD and retains the remote heap * information across daemon restarts. Therefore only * the first attach should provide heap information to * DSP. Subsequent attaches reuse the previously * initialized memory pool. */ spin_lock_irqsave(&cctx->lock, flags); if (!cctx->audio_init_mem) { pages[0].addr = cctx->remote_heap_addr; pages[0].size = cctx->remote_heap_size; cctx->audio_init_mem = true; inbuf.pageslen = 1; sent_heap = true; } else { pages[0].addr = 0; pages[0].size = 0; } spin_unlock_irqrestore(&cctx->lock, flags);
args[2].ptr = (u64)(uintptr_t) pages; args[2].length = sizeof(*pages); args[2].fd = -1;
sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, args); if (err) goto err_invoke;
kfree(args); kfree(name);
return 0;err_invoke: if (sent_heap) { spin_lock_irqsave(&cctx->lock, flags); cctx->audio_init_mem = false; spin_unlock_irqrestore(&cctx->lock, flags); } kfree(name);err: kfree(args);
return err;}
static int fastrpc_init_create_process(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_init_create init; struct fastrpc_invoke_args *args; struct fastrpc_phy_page pages[1]; struct fastrpc_map *map = NULL; struct fastrpc_buf *imem = NULL; int memlen; int err; struct { int client_id; u32 namelen; u32 filelen; u32 pageslen; u32 attrs; u32 siglen; } inbuf; u32 sc; bool unsigned_module = false;
args = kzalloc_objs(*args, FASTRPC_CREATE_PROCESS_NARGS); if (!args) return -ENOMEM;
if (copy_from_user(&init, argp, sizeof(init))) { err = -EFAULT; goto err; }
if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE) unsigned_module = true;
if (is_session_rejected(fl, unsigned_module)) { err = -ECONNREFUSED; goto err; }
if (init.filelen > INIT_FILELEN_MAX) { err = -EINVAL; goto err; }
inbuf.client_id = fl->client_id; inbuf.namelen = strlen(current->comm) + 1; inbuf.filelen = init.filelen; inbuf.pageslen = 1; inbuf.attrs = init.attrs; inbuf.siglen = init.siglen; fl->pd = USER_PD;
if (init.filelen && init.filefd) { err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map); if (err) goto err; }
memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4), 1024 * 1024); err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen, &imem); if (err) goto err_alloc;
fl->init_mem = imem; args[0].ptr = (u64)(uintptr_t)&inbuf; args[0].length = sizeof(inbuf); args[0].fd = -1;
args[1].ptr = (u64)(uintptr_t)current->comm; args[1].length = inbuf.namelen; args[1].fd = -1;
args[2].ptr = (u64) init.file; args[2].length = inbuf.filelen; args[2].fd = init.filefd;
pages[0].addr = imem->dma_addr; pages[0].size = imem->size;
args[3].ptr = (u64)(uintptr_t) pages; args[3].length = 1 * sizeof(*pages); args[3].fd = -1;
args[4].ptr = (u64)(uintptr_t)&inbuf.attrs; args[4].length = sizeof(inbuf.attrs); args[4].fd = -1;
args[5].ptr = (u64)(uintptr_t) &inbuf.siglen; args[5].length = sizeof(inbuf.siglen); args[5].fd = -1;
sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0); if (init.attrs) sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, args); if (err) goto err_invoke;
kfree(args);
return 0;
err_invoke: fl->init_mem = NULL; fastrpc_buf_free(imem);err_alloc: fastrpc_map_put(map);err: kfree(args);
return err;}
static struct fastrpc_session_ctx *fastrpc_session_alloc( struct fastrpc_user *fl){ struct fastrpc_channel_ctx *cctx = fl->cctx; struct fastrpc_session_ctx *session = NULL; unsigned long flags; int i;
spin_lock_irqsave(&cctx->lock, flags); for (i = 0; i < cctx->sesscount; i++) { if (!cctx->session[i].used && cctx->session[i].valid) { cctx->session[i].used = true; session = &cctx->session[i]; /* any non-zero ID will work, session_idx + 1 is the simplest one */ fl->client_id = i + 1; break; } } spin_unlock_irqrestore(&cctx->lock, flags);
return session;}
static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx, struct fastrpc_session_ctx *session){ unsigned long flags;
spin_lock_irqsave(&cctx->lock, flags); session->used = false; spin_unlock_irqrestore(&cctx->lock, flags);}
static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl){ struct fastrpc_invoke_args args[1]; int client_id = 0; u32 sc;
client_id = fl->client_id; args[0].ptr = (u64)(uintptr_t) &client_id; args[0].length = sizeof(client_id); args[0].fd = -1; sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);}
static int fastrpc_device_release(struct inode *inode, struct file *file){ struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data; struct fastrpc_channel_ctx *cctx = fl->cctx; unsigned long flags;
fastrpc_release_current_dsp_process(fl);
spin_lock_irqsave(&cctx->lock, flags); list_del(&fl->user); spin_unlock_irqrestore(&cctx->lock, flags);
fastrpc_session_free(cctx, fl->sctx); file->private_data = NULL; /* Release the reference taken in fastrpc_device_open */ fastrpc_user_put(fl);
return 0;}
static int fastrpc_device_open(struct inode *inode, struct file *filp){ struct fastrpc_channel_ctx *cctx; struct fastrpc_device *fdevice; struct fastrpc_user *fl = NULL; unsigned long flags;
fdevice = miscdev_to_fdevice(filp->private_data); cctx = fdevice->cctx;
fl = kzalloc_obj(*fl); if (!fl) return -ENOMEM;
/* Released in fastrpc_device_release() */ fastrpc_channel_ctx_get(cctx);
filp->private_data = fl; spin_lock_init(&fl->lock); mutex_init(&fl->mutex); INIT_LIST_HEAD(&fl->pending); INIT_LIST_HEAD(&fl->maps); INIT_LIST_HEAD(&fl->mmaps); INIT_LIST_HEAD(&fl->user); fl->cctx = cctx; fl->is_secure_dev = fdevice->secure;
fl->sctx = fastrpc_session_alloc(fl); if (!fl->sctx) { dev_err_ratelimited(&cctx->rpdev->dev, "No session available\n"); mutex_destroy(&fl->mutex); kfree(fl); fastrpc_channel_ctx_put(cctx); return -EBUSY; }
spin_lock_irqsave(&cctx->lock, flags); list_add_tail(&fl->user, &cctx->users); spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&fl->refcount);
return 0;}
static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_alloc_dma_buf bp; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); struct fastrpc_buf *buf = NULL; int err;
if (copy_from_user(&bp, argp, sizeof(bp))) return -EFAULT;
err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf); if (err) return err; exp_info.ops = &fastrpc_dma_buf_ops; exp_info.size = bp.size; exp_info.flags = O_RDWR; exp_info.priv = buf; buf->dmabuf = dma_buf_export(&exp_info); if (IS_ERR(buf->dmabuf)) { err = PTR_ERR(buf->dmabuf); fastrpc_buf_free(buf); return err; }
bp.fd = get_unused_fd_flags(O_ACCMODE); if (bp.fd < 0) { dma_buf_put(buf->dmabuf); return bp.fd; }
if (copy_to_user(argp, &bp, sizeof(bp))) { put_unused_fd(bp.fd); dma_buf_put(buf->dmabuf); return -EFAULT; }
dma_buf_fd_install(buf->dmabuf, bp.fd);
return 0;}
static int fastrpc_init_attach(struct fastrpc_user *fl, int pd){ struct fastrpc_invoke_args args[1]; int client_id = fl->client_id; u32 sc;
args[0].ptr = (u64)(uintptr_t) &client_id; args[0].length = sizeof(client_id); args[0].fd = -1; sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0); fl->pd = pd;
return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);}
static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_invoke_args *args = NULL; struct fastrpc_invoke inv; u32 nscalars; int err;
if (copy_from_user(&inv, argp, sizeof(inv))) return -EFAULT;
/* nscalars is truncated here to max supported value */ nscalars = REMOTE_SCALARS_LENGTH(inv.sc); if (nscalars) { args = kzalloc_objs(*args, nscalars); if (!args) return -ENOMEM;
if (copy_from_user(args, (void __user *)(uintptr_t)inv.args, nscalars * sizeof(*args))) { kfree(args); return -EFAULT; } }
err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args); kfree(args);
return err;}
static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf, uint32_t dsp_attr_buf_len){ struct fastrpc_invoke_args args[2] = { 0 };
/* * Capability filled in userspace. This carries the information * about the remoteproc support which is fetched from the remoteproc * sysfs node by userspace. */ dsp_attr_buf[0] = 0; dsp_attr_buf_len -= 1;
args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len; args[0].length = sizeof(dsp_attr_buf_len); args[0].fd = -1; args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1]; args[1].length = dsp_attr_buf_len * sizeof(u32); args[1].fd = -1;
return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE, FASTRPC_SCALARS(0, 1, 1), args);}
static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap, struct fastrpc_user *fl){ struct fastrpc_channel_ctx *cctx = fl->cctx; uint32_t attribute_id = cap->attribute_id; uint32_t *dsp_attributes; unsigned long flags; int err;
spin_lock_irqsave(&cctx->lock, flags); /* check if we already have queried dsp for attributes */ if (cctx->valid_attributes) { spin_unlock_irqrestore(&cctx->lock, flags); goto done; } spin_unlock_irqrestore(&cctx->lock, flags);
dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL); if (!dsp_attributes) return -ENOMEM;
err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES); if (err == DSP_UNSUPPORTED_API) { dev_info(&cctx->rpdev->dev, "Warning: DSP capabilities not supported\n"); kfree(dsp_attributes); return -EOPNOTSUPP; } else if (err) { dev_dbg(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err); kfree(dsp_attributes); return err; }
spin_lock_irqsave(&cctx->lock, flags); memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN); cctx->valid_attributes = true; spin_unlock_irqrestore(&cctx->lock, flags); kfree(dsp_attributes);done: cap->capability = cctx->dsp_attributes[attribute_id]; return 0;}
static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_ioctl_set_option opt = {0}; int i;
if (copy_from_user(&opt, argp, sizeof(opt))) return -EFAULT;
for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) { if (opt.reserved[i] != 0) return -EINVAL; }
if (opt.request_id != FASTRPC_POLL_MODE) return -EINVAL;
if (!fl->cctx->poll_mode_supported) return -EOPNOTSUPP;
if (opt.value == FASTRPC_POLL_MODE_ENABLE) fl->poll_mode = true; else if (opt.value == FASTRPC_POLL_MODE_DISABLE) fl->poll_mode = false; else return -EINVAL;
return 0;}
static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_ioctl_capability cap = {0}; int err = 0;
if (copy_from_user(&cap, argp, sizeof(cap))) return -EFAULT;
cap.capability = 0;
if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) { dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n", cap.attribute_id, err); return -EOVERFLOW; }
err = fastrpc_get_info_from_kernel(&cap, fl); if (err) return err;
if (copy_to_user(argp, &cap, sizeof(cap))) return -EFAULT;
return 0;}
static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf){ struct fastrpc_invoke_args args[1] = { [0] = { 0 } }; struct fastrpc_munmap_req_msg req_msg; struct device *dev = fl->sctx->dev; int err; u32 sc;
req_msg.client_id = fl->client_id; req_msg.size = buf->size; req_msg.vaddr = buf->raddr;
args[0].ptr = (u64) (uintptr_t) &req_msg; args[0].length = sizeof(req_msg);
sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0); err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]); if (!err) { dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); fastrpc_buf_free(buf); } else { dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); }
return err;}
static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_buf *buf = NULL, *iter, *b; struct fastrpc_req_munmap req; struct device *dev = fl->sctx->dev; int err;
if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT;
spin_lock(&fl->lock); list_for_each_entry_safe(iter, b, &fl->mmaps, node) { if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) { list_del(&iter->node); buf = iter; break; } } spin_unlock(&fl->lock);
if (!buf) { dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n", req.vaddrout, req.size); return -EINVAL; }
err = fastrpc_req_munmap_impl(fl, buf); if (err) { spin_lock(&fl->lock); list_add_tail(&buf->node, &fl->mmaps); spin_unlock(&fl->lock); }
return err;}
static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } }; struct fastrpc_buf *buf = NULL; struct fastrpc_mmap_req_msg req_msg; struct fastrpc_mmap_rsp_msg rsp_msg; struct fastrpc_phy_page pages; struct fastrpc_req_mmap req; struct device *dev = fl->sctx->dev; int err; u32 sc;
if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT;
if (req.flags != ADSP_MMAP_ADD_PAGES) { dev_err(dev, "flag not supported 0x%x\n", req.flags);
return -EINVAL; }
if (req.vaddrin) { dev_err(dev, "adding user allocated pages is not supported\n"); return -EINVAL; }
err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
if (err) { dev_err(dev, "failed to allocate buffer\n"); return err; }
req_msg.client_id = fl->client_id; req_msg.flags = req.flags; req_msg.vaddr = req.vaddrin; req_msg.num = sizeof(pages);
args[0].ptr = (u64) (uintptr_t) &req_msg; args[0].length = sizeof(req_msg);
pages.addr = buf->dma_addr; pages.size = buf->size;
args[1].ptr = (u64) (uintptr_t) &pages; args[1].length = sizeof(pages);
args[2].ptr = (u64) (uintptr_t) &rsp_msg; args[2].length = sizeof(rsp_msg);
sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1); err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]); if (err) { dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size); fastrpc_buf_free(buf); return err; }
/* update the buffer to be able to deallocate the memory on the DSP */ buf->raddr = (uintptr_t) rsp_msg.vaddr;
/* let the client know the address to use */ req.vaddrout = rsp_msg.vaddr;
spin_lock(&fl->lock); list_add_tail(&buf->node, &fl->mmaps); spin_unlock(&fl->lock);
if (copy_to_user((void __user *)argp, &req, sizeof(req))) { err = -EFAULT; goto err_assign; }
dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n", buf->raddr, buf->size);
return 0;
err_assign: fastrpc_req_munmap_impl(fl, buf);
return err;}
static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req){ struct fastrpc_invoke_args args[1] = { [0] = { 0 } }; struct fastrpc_map *map = NULL, *iter, *m; struct fastrpc_mem_unmap_req_msg req_msg = { 0 }; int err = 0; u32 sc; struct device *dev = fl->sctx->dev;
spin_lock(&fl->lock); list_for_each_entry_safe(iter, m, &fl->maps, node) { if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) { map = iter; break; } }
spin_unlock(&fl->lock);
if (!map) { dev_err(dev, "map not in list\n"); return -EINVAL; }
req_msg.client_id = fl->client_id; req_msg.len = map->len; req_msg.vaddrin = map->raddr; req_msg.fd = map->fd;
args[0].ptr = (u64) (uintptr_t) &req_msg; args[0].length = sizeof(req_msg);
sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0); err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]); if (err) { dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr); return err; } fastrpc_map_put(map);
return 0;}
static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_mem_unmap req;
if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT;
return fastrpc_req_mem_unmap_impl(fl, &req);}
static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp){ struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } }; struct fastrpc_mem_map_req_msg req_msg = { 0 }; struct fastrpc_mmap_rsp_msg rsp_msg = { 0 }; struct fastrpc_mem_unmap req_unmap = { 0 }; struct fastrpc_phy_page pages = { 0 }; struct fastrpc_mem_map req; struct device *dev = fl->sctx->dev; struct fastrpc_map *map = NULL; int err; u32 sc;
if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT;
/* create SMMU mapping */ err = fastrpc_map_create(fl, req.fd, req.length, 0, &map); if (err) { dev_err(dev, "failed to map buffer, fd = %d\n", req.fd); return err; }
req_msg.client_id = fl->client_id; req_msg.fd = req.fd; req_msg.offset = req.offset; req_msg.vaddrin = req.vaddrin; map->va = (void *) (uintptr_t) req.vaddrin; req_msg.flags = req.flags; req_msg.num = sizeof(pages); req_msg.data_len = 0;
args[0].ptr = (u64) (uintptr_t) &req_msg; args[0].length = sizeof(req_msg);
pages.addr = map->dma_addr; pages.size = map->len;
args[1].ptr = (u64) (uintptr_t) &pages; args[1].length = sizeof(pages);
args[2].ptr = (u64) (uintptr_t) &pages; args[2].length = 0;
args[3].ptr = (u64) (uintptr_t) &rsp_msg; args[3].length = sizeof(rsp_msg);
sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1); err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]); if (err) { dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n", req.fd, req.vaddrin, map->len); goto err_invoke; }
/* update the buffer to be able to deallocate the memory on the DSP */ map->raddr = rsp_msg.vaddr;
/* let the client know the address to use */ req.vaddrout = rsp_msg.vaddr;
if (copy_to_user((void __user *)argp, &req, sizeof(req))) { /* unmap the memory and release the buffer */ req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr; req_unmap.length = map->len; fastrpc_req_mem_unmap_impl(fl, &req_unmap); return -EFAULT; }
return 0;
err_invoke: fastrpc_map_put(map);
return err;}
static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, unsigned long arg){ struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data; char __user *argp = (char __user *)arg; int err;
switch (cmd) { case FASTRPC_IOCTL_INVOKE: err = fastrpc_invoke(fl, argp); break; case FASTRPC_IOCTL_INIT_ATTACH: err = fastrpc_init_attach(fl, ROOT_PD); break; case FASTRPC_IOCTL_INIT_ATTACH_SNS: err = fastrpc_init_attach(fl, SENSORS_PD); break; case FASTRPC_IOCTL_INIT_CREATE_STATIC: err = fastrpc_init_create_static_process(fl, argp); break; case FASTRPC_IOCTL_INIT_CREATE: err = fastrpc_init_create_process(fl, argp); break; case FASTRPC_IOCTL_ALLOC_DMA_BUFF: err = fastrpc_dmabuf_alloc(fl, argp); break; case FASTRPC_IOCTL_MMAP: err = fastrpc_req_mmap(fl, argp); break; case FASTRPC_IOCTL_MUNMAP: err = fastrpc_req_munmap(fl, argp); break; case FASTRPC_IOCTL_MEM_MAP: err = fastrpc_req_mem_map(fl, argp); break; case FASTRPC_IOCTL_MEM_UNMAP: err = fastrpc_req_mem_unmap(fl, argp); break; case FASTRPC_IOCTL_SET_OPTION: err = fastrpc_set_option(fl, argp); break; case FASTRPC_IOCTL_GET_DSP_INFO: err = fastrpc_get_dsp_info(fl, argp); break; default: err = -ENOTTY; break; }
return err;}
static const struct file_operations fastrpc_fops = { .open = fastrpc_device_open, .release = fastrpc_device_release, .unlocked_ioctl = fastrpc_device_ioctl, .compat_ioctl = fastrpc_device_ioctl,};
static int fastrpc_cb_probe(struct platform_device *pdev){ struct fastrpc_channel_ctx *cctx; struct fastrpc_session_ctx *sess; struct device *dev = &pdev->dev; int i, sessions = 0; unsigned long flags; u32 dma_bits; u32 sid = 0; int rc;
cctx = dev_get_drvdata(dev->parent); if (!cctx) return -EINVAL;
of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions); if (of_property_read_u32(dev->of_node, "reg", &sid)) dev_info(dev, "FastRPC Session ID not specified in DT\n");
spin_lock_irqsave(&cctx->lock, flags); if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) { spin_unlock_irqrestore(&cctx->lock, flags); dev_err(&pdev->dev, "too many sessions\n"); return -ENOSPC; } dma_bits = cctx->soc_data->dma_addr_bits_default; sess = &cctx->session[cctx->sesscount++]; sess->used = false; sess->valid = true; sess->dev = dev; dev_set_drvdata(dev, sess); sess->sid = sid;
if (cctx->domain_id == CDSP_DOMAIN_ID) dma_bits = cctx->soc_data->dma_addr_bits_cdsp;
if (sessions > 0) { struct fastrpc_session_ctx *dup_sess;
for (i = 1; i < sessions; i++) { if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) break; dup_sess = &cctx->session[cctx->sesscount++]; memcpy(dup_sess, sess, sizeof(*dup_sess)); } } spin_unlock_irqrestore(&cctx->lock, flags); rc = dma_set_mask(dev, DMA_BIT_MASK(dma_bits)); if (rc) { dev_err(dev, "%u-bit DMA enable failed\n", dma_bits); return rc; }
return 0;}
static void fastrpc_cb_remove(struct platform_device *pdev){ struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent); struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev); unsigned long flags; int i;
spin_lock_irqsave(&cctx->lock, flags); for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) { if (cctx->session[i].sid == sess->sid) { cctx->session[i].valid = false; cctx->sesscount--; } } spin_unlock_irqrestore(&cctx->lock, flags);}
static const struct of_device_id fastrpc_match_table[] = { { .compatible = "qcom,fastrpc-compute-cb" }, { }};
static struct platform_driver fastrpc_cb_driver = { .probe = fastrpc_cb_probe, .remove = fastrpc_cb_remove, .driver = { .name = "qcom,fastrpc-cb", .of_match_table = fastrpc_match_table, .suppress_bind_attrs = true, },};
static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx, bool is_secured, const char *domain){ struct fastrpc_device *fdev; int err;
fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL); if (!fdev) return -ENOMEM;
fdev->secure = is_secured; fdev->cctx = cctx; fdev->miscdev.minor = MISC_DYNAMIC_MINOR; fdev->miscdev.fops = &fastrpc_fops; fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s", domain, is_secured ? "-secure" : ""); if (!fdev->miscdev.name) return -ENOMEM;
err = misc_register(&fdev->miscdev); if (!err) { if (is_secured) cctx->secure_fdevice = fdev; else cctx->fdevice = fdev; }
return err;}
static int fastrpc_get_domain_id(const char *domain){ if (!strncmp(domain, "adsp", 4)) return ADSP_DOMAIN_ID; else if (!strncmp(domain, "cdsp", 4)) return CDSP_DOMAIN_ID; else if (!strncmp(domain, "mdsp", 4)) return MDSP_DOMAIN_ID; else if (!strncmp(domain, "sdsp", 4)) return SDSP_DOMAIN_ID; else if (!strncmp(domain, "gdsp", 4)) return GDSP_DOMAIN_ID;
return -EINVAL;}
static const struct fastrpc_soc_data kaanapali_soc_data = { .sid_pos = 56, .dma_addr_bits_cdsp = 34, .dma_addr_bits_default = 32, .poll_mode_supported = true,};
static const struct fastrpc_soc_data default_soc_data = { .sid_pos = 32, .dma_addr_bits_cdsp = 32, .dma_addr_bits_default = 32,};
/* * Exception list for older platforms that use default_soc_data but whose * DSP firmware supports FastRPC polling mode. * * NOTE: This list is intentionally closed. * Do NOT add new platforms here. New SoCs must advertise polling mode * support via their soc_data. */
static const struct of_device_id fastrpc_poll_supported_machines[] __maybe_unused = { { .compatible = "qcom,milos" }, { .compatible = "qcom,qcs8300" }, { .compatible = "qcom,sa8775p" }, { .compatible = "qcom,sar2130p" }, { .compatible = "qcom,sm8450" }, { .compatible = "qcom,sm8550" }, { .compatible = "qcom,sm8650" }, { .compatible = "qcom,sm8750" }, { .compatible = "qcom,x1e80100" }, { .compatible = "qcom,x1p42100" }, {},};
static int fastrpc_init_reserved_mem(struct fastrpc_channel_ctx *cctx, struct device *rdev, u32 domain_id){ struct resource res; u64 src_perms; int err;
/* Only SDSP and ADSP domains use a reserved remote heap region */ if (domain_id != SDSP_DOMAIN_ID && domain_id != ADSP_DOMAIN_ID) return 0;
err = of_reserved_mem_region_to_resource(rdev->of_node, 0, &res); if (err) { /* * The reserved-memory region is optional at probe time. For * the Audio PD (ADSP) domain its absence is validated later in * fastrpc_init_create_static_process(), so keep probing here. */ return 0; }
/* * Audio PD (ADSP) is a static PD: cache the region so its lifetime * is tied to the rpmsg channel instead of being controlled by * userspace. */ if (domain_id == ADSP_DOMAIN_ID) { cctx->remote_heap_addr = res.start; cctx->remote_heap_size = resource_size(&res); }
if (!cctx->vmcount) return 0;
src_perms = BIT(QCOM_SCM_VMID_HLOS); return qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms, cctx->vmperms, cctx->vmcount);}
static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev){ struct device *rdev = &rpdev->dev; struct fastrpc_channel_ctx *data; int i, err, domain_id = -1, vmcount; const char *domain; bool secure_dsp; unsigned int vmids[FASTRPC_MAX_VMIDS]; const struct fastrpc_soc_data *soc_data;
soc_data = device_get_match_data(rdev);
err = of_property_read_string(rdev->of_node, "label", &domain); if (err) { dev_info(rdev, "FastRPC Domain not specified in DT\n"); return err; }
domain_id = fastrpc_get_domain_id(domain);
if (domain_id < 0) { dev_info(rdev, "FastRPC Domain %s not supported\n", domain); return -EINVAL; }
if (devm_of_reserved_mem_device_init(rdev)) dev_dbg(rdev, "no reserved DMA memory for FASTRPC\n");
vmcount = of_property_read_variable_u32_array(rdev->of_node, "qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS); if (vmcount < 0) vmcount = 0; else if (!qcom_scm_is_available()) return -EPROBE_DEFER;
data = kzalloc_obj(*data); if (!data) return -ENOMEM;
if (vmcount) { data->vmcount = vmcount; for (i = 0; i < data->vmcount; i++) { data->vmperms[i].vmid = vmids[i]; data->vmperms[i].perm = QCOM_SCM_PERM_RWX; } }
err = fastrpc_init_reserved_mem(data, rdev, domain_id); if (err) goto err_free_data;
secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); data->secure = secure_dsp; data->soc_data = soc_data; data->poll_mode_supported = soc_data->poll_mode_supported || of_machine_get_match(fastrpc_poll_supported_machines);
switch (domain_id) { case ADSP_DOMAIN_ID: case MDSP_DOMAIN_ID: case SDSP_DOMAIN_ID: /* Unsigned PD offloading is only supported on CDSP and GDSP */ data->unsigned_support = false; err = fastrpc_device_register(rdev, data, secure_dsp, domain); if (err) goto err_free_data; break; case CDSP_DOMAIN_ID: case GDSP_DOMAIN_ID: data->unsigned_support = true; /* Create both device nodes so that we can allow both Signed and Unsigned PD */ err = fastrpc_device_register(rdev, data, true, domain); if (err) goto err_free_data;
err = fastrpc_device_register(rdev, data, false, domain); if (err) goto err_deregister_fdev; break; default: err = -EINVAL; goto err_free_data; }
kref_init(&data->refcount); atomic_set(&data->ctx_seq, 0);
rdev->dma_mask = &data->dma_mask; dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32)); INIT_LIST_HEAD(&data->users); INIT_LIST_HEAD(&data->invoke_interrupted_mmaps); spin_lock_init(&data->lock); idr_init(&data->ctx_idr); data->domain_id = domain_id; data->rpdev = rpdev; dev_set_drvdata(&rpdev->dev, data);
err = of_platform_populate(rdev->of_node, NULL, NULL, rdev); if (err) goto err_deregister_fdev;
return 0;
err_deregister_fdev: if (data->fdevice) misc_deregister(&data->fdevice->miscdev); if (data->secure_fdevice) misc_deregister(&data->secure_fdevice->miscdev);
err_free_data: kfree(data); return err;}
static void fastrpc_notify_users(struct fastrpc_user *user){ struct fastrpc_invoke_ctx *ctx;
spin_lock(&user->lock); list_for_each_entry(ctx, &user->pending, node) { ctx->retval = -EPIPE; complete(&ctx->work); } spin_unlock(&user->lock);}
static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev){ struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); struct fastrpc_buf *buf, *b; struct fastrpc_user *user; unsigned long flags;
/* No invocations past this point */ spin_lock_irqsave(&cctx->lock, flags); cctx->rpdev = NULL; list_for_each_entry(user, &cctx->users, user) fastrpc_notify_users(user); spin_unlock_irqrestore(&cctx->lock, flags);
if (cctx->fdevice) misc_deregister(&cctx->fdevice->miscdev);
if (cctx->secure_fdevice) misc_deregister(&cctx->secure_fdevice->miscdev);
list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node) list_del(&buf->node);
if (cctx->remote_heap_size && cctx->vmcount) { u64 src_perms = 0; int err, i; struct qcom_scm_vmperm dst_perms;
for (i = 0; i < cctx->vmcount; i++) src_perms |= BIT(cctx->vmperms[i].vmid);
dst_perms.vmid = QCOM_SCM_VMID_HLOS; dst_perms.perm = QCOM_SCM_PERM_RWX;
err = qcom_scm_assign_mem(cctx->remote_heap_addr, cctx->remote_heap_size, &src_perms, &dst_perms, 1); if (err) dev_err(&rpdev->dev, "Failed to assign memory back to HLOS: addr %pa size %#llx err %d\n", &cctx->remote_heap_addr, cctx->remote_heap_size, err); }
of_platform_depopulate(&rpdev->dev);
fastrpc_channel_ctx_put(cctx);}
static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, int len, void *priv, u32 addr){ struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); struct fastrpc_invoke_rsp *rsp = data; struct fastrpc_invoke_ctx *ctx; unsigned long flags; unsigned long ctxid;
if (len < sizeof(*rsp)) return -EINVAL;
if (!cctx) return -ENODEV;
/* * A PD exit notification from the DSP PD notification framework carries * this sentinel rather than a real context. Drop it: a real context is * (idr_index << 4) | pd and can never collide with this value. */ if (rsp->ctx == FASTRPC_DSP_PD_NOTIFY_CTX) return 0;
ctxid = FIELD_GET(FASTRPC_CTXID_MASK, rsp->ctx);
spin_lock_irqsave(&cctx->lock, flags); ctx = idr_find(&cctx->ctx_idr, ctxid); spin_unlock_irqrestore(&cctx->lock, flags);
if (!ctx) { dev_err(&rpdev->dev, "No context ID matches response\n"); return -ENOENT; }
/* * Validate the sequence number embedded in the upper bits of the * context ID. Under high concurrency the IDR slot can be recycled * for a new request before a late (or duplicate) glink COMPLETE * response for the previous request arrives. Without this check the * driver would call complete() on the wrong context, waking a thread * whose buffers are still being accessed by the DSP. */ if (FIELD_GET(FASTRPC_CTXID_SEQ_MASK, rsp->ctx) != FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)) { dev_dbg(&rpdev->dev, "Stale glink response ctx 0x%llx (expected seq 0x%llx), dropping\n", rsp->ctx, FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)); return 0; }
ctx->retval = rsp->retval; ctx->is_work_done = true; complete(&ctx->work);
/* * The DMA buffer associated with the context cannot be freed in * interrupt context so schedule it through a worker thread to * avoid a kernel BUG. */ schedule_work(&ctx->put_work);
return 0;}
static const struct of_device_id fastrpc_rpmsg_of_match[] = { { .compatible = "qcom,kaanapali-fastrpc", .data = &kaanapali_soc_data }, { .compatible = "qcom,fastrpc", .data = &default_soc_data }, { }};MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
static struct rpmsg_driver fastrpc_driver = { .probe = fastrpc_rpmsg_probe, .remove = fastrpc_rpmsg_remove, .callback = fastrpc_rpmsg_callback, .drv = { .name = "qcom,fastrpc", .of_match_table = fastrpc_rpmsg_of_match, },};
static int fastrpc_init(void){ int ret;
ret = platform_driver_register(&fastrpc_cb_driver); if (ret < 0) { pr_err("fastrpc: failed to register cb driver\n"); return ret; }
ret = register_rpmsg_driver(&fastrpc_driver); if (ret < 0) { pr_err("fastrpc: failed to register rpmsg driver\n"); platform_driver_unregister(&fastrpc_cb_driver); return ret; }
return 0;}module_init(fastrpc_init);
static void fastrpc_exit(void){ platform_driver_unregister(&fastrpc_cb_driver); unregister_rpmsg_driver(&fastrpc_driver);}module_exit(fastrpc_exit);
MODULE_DESCRIPTION("Qualcomm FastRPC");MODULE_LICENSE("GPL v2");MODULE_IMPORT_NS("DMA_BUF");