Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
#include <linux/module.h>#include <linux/printk.h>#include <rdma/ib_addr.h>#include <rdma/ib_cache.h>#include <rdma/ib_user_verbs.h>#include <ionic_api.h>
#include "ionic_fw.h"#include "ionic_ibdev.h"
#define ionic_set_ecn(tos) (((tos) | 2u) & ~1u)#define ionic_clear_ecn(tos) ((tos) & ~3u)
static int ionic_validate_qdesc(struct ionic_qdesc *q){ if (!q->addr || !q->size || !q->mask || !q->depth_log2 || !q->stride_log2) return -EINVAL;
if (q->addr & (PAGE_SIZE - 1)) return -EINVAL;
if (q->mask != BIT(q->depth_log2) - 1) return -EINVAL;
if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2)) return -EINVAL;
return 0;}
static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx){ /* EQ per vector per udma, and the first eqs reserved for async events. * The rest of the vectors can be requested for completions. */ u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1;
return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx;}
static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx){ unsigned int size, base, bound; int rc;
size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count; base = size * udma_idx; bound = base + size;
rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound); if (rc >= 0) { /* cq_base is zero or a multiple of two queue groups */ *cqid = dev->lif_cfg.cq_base + ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift, dev->half_cqid_udma_shift);
rc = 0; }
return rc;}
static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid){ u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base, dev->lif_cfg.udma_qgrp_shift, dev->half_cqid_udma_shift);
ionic_resid_put(&dev->inuse_cqid, bitid);}
int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_tbl_buf *buf, const struct ib_cq_init_attr *attr, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, int udma_idx){ struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); struct ionic_cq *cq = &vcq->cq[udma_idx]; int rc;
cq->vcq = vcq;
if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) { rc = -EINVAL; goto err_args; }
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); if (rc) goto err_args; }
cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx);
spin_lock_init(&cq->lock); INIT_LIST_HEAD(&cq->poll_sq); INIT_LIST_HEAD(&cq->flush_sq); INIT_LIST_HEAD(&cq->flush_rq);
if (udata) { rc = ionic_validate_qdesc(req_cq); if (rc) goto err_qdesc;
cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, req_cq->addr, req_cq->size, IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { rc = PTR_ERR(cq->umem); goto err_qdesc; }
cq->q.ptr = NULL; cq->q.size = req_cq->size; cq->q.mask = req_cq->mask; cq->q.depth_log2 = req_cq->depth_log2; cq->q.stride_log2 = req_cq->stride_log2; } else { rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev, attr->cqe + IONIC_CQ_GRACE, sizeof(struct ionic_v1_cqe)); if (rc) goto err_q_init;
cq->color = true; cq->credit = cq->q.mask; }
rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE); if (rc) goto err_pgtbl_init;
init_completion(&cq->cq_rel_comp); kref_init(&cq->cq_kref);
return 0;
err_pgtbl_init: if (!udata) ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);err_q_init: if (cq->umem) ib_umem_release(cq->umem);err_qdesc: if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) ionic_put_cqid(dev, cq->cqid);err_args: cq->vcq = NULL;
return rc;}
int ionic_post_create_cq_cmd(struct ionic_cq *cq, struct ib_udata *udata, __u32 *resp_cqid){ struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device);
if (udata && resp_cqid) *resp_cqid = cq->cqid; else ionic_queue_dbell_init(&cq->q, cq->cqid);
return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);}
void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq){ if (!cq->vcq) return;
if (!xa_erase_irq(&dev->cq_tbl, cq->cqid)) return;
kref_put(&cq->cq_kref, ionic_cq_complete); wait_for_completion(&cq->cq_rel_comp);}
void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq){ if (!cq->vcq) return;
if (cq->umem) ib_umem_release(cq->umem); else ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) ionic_put_cqid(dev, cq->cqid);
cq->vcq = NULL;}
static int ionic_validate_qdesc_zero(struct ionic_qdesc *q){ if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2) return -EINVAL;
return 0;}
static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid){ int rc;
rc = ionic_resid_get(&dev->inuse_pdid); if (rc < 0) return rc;
*pdid = rc; return 0;}
static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid){ int rc;
rc = ionic_resid_get(&dev->inuse_ahid); if (rc < 0) return rc;
*ahid = rc; return 0;}
static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid){ int rc;
/* wrap to 1, skip reserved lkey */ rc = ionic_resid_get_shared(&dev->inuse_mrid, 1, dev->inuse_mrid.inuse_size); if (rc < 0) return rc;
*mrid = ionic_mrid(rc, dev->next_mrkey++); return 0;}
static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid){ int rc = 0;
rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1); if (rc < 0) return rc;
*qpid = IB_QPT_GSI; return 0;}
static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid, u8 *udma_idx, u8 udma_mask){ unsigned int size, base, bound; int udma_i, udma_x, udma_ix; int rc = -EINVAL;
udma_x = dev->next_qpid_udma_idx;
dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1;
for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) { udma_ix = udma_i ^ udma_x;
if (!(udma_mask & BIT(udma_ix))) continue;
size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count; base = size * udma_ix; bound = base + size;
/* skip reserved SMI and GSI qpids in group zero */ if (!base) base = 2;
rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound); if (rc >= 0) { *qpid = ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift, dev->half_qpid_udma_shift); *udma_idx = udma_ix;
rc = 0; break; } }
return rc;}
static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr){ int rc, dbpage_num;
/* wrap to 1, skip kernel reserved */ rc = ionic_resid_get_shared(&dev->inuse_dbid, 1, dev->inuse_dbid.inuse_size); if (rc < 0) return rc;
dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc; *addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT);
*dbid = rc;
return 0;}
static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid){ ionic_resid_put(&dev->inuse_pdid, pdid);}
static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid){ ionic_resid_put(&dev->inuse_ahid, ahid);}
static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid){ ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid));}
static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid){ u32 bitid = ionic_qid_to_bitid(qpid, dev->lif_cfg.udma_qgrp_shift, dev->half_qpid_udma_shift);
ionic_resid_put(&dev->inuse_qpid, bitid);}
static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid){ ionic_resid_put(&dev->inuse_dbid, dbid);}
static struct rdma_user_mmap_entry*ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size, unsigned long pfn, u8 mmap_flags, u64 *offset){ struct ionic_mmap_entry *entry; int rc;
entry = kzalloc_obj(*entry); if (!entry) return NULL;
entry->size = size; entry->pfn = pfn; entry->mmap_flags = mmap_flags;
rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry, entry->size); if (rc) { kfree(entry); return NULL; }
if (offset) *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
return &entry->rdma_entry;}
int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); struct ionic_ctx *ctx = to_ionic_ctx(ibctx); struct ionic_ctx_resp resp = {}; struct ionic_ctx_req req; phys_addr_t db_phys = 0; int rc;
rc = ib_copy_validate_udata_in(udata, req, rsvd); if (rc) return rc;
/* try to allocate dbid for user ctx */ rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys); if (rc < 0) return rc;
ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid);
ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE, PHYS_PFN(db_phys), 0, NULL); if (!ctx->mmap_dbell) { rc = -ENOMEM; goto err_mmap_dbell; }
if (dev->lif_cfg.phc_state) { ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0, IONIC_MMAP_PHC, &resp.phc_offset); if (!ctx->mmap_phc) { rc = -ENOMEM; goto err_mmap_phc; } }
resp.page_shift = PAGE_SHIFT;
resp.dbell_offset = db_phys & ~PAGE_MASK;
resp.version = dev->lif_cfg.rdma_version; resp.qp_opcodes = dev->lif_cfg.qp_opcodes; resp.admin_opcodes = dev->lif_cfg.admin_opcodes;
resp.sq_qtype = dev->lif_cfg.sq_qtype; resp.rq_qtype = dev->lif_cfg.rq_qtype; resp.cq_qtype = dev->lif_cfg.cq_qtype; resp.admin_qtype = dev->lif_cfg.aq_qtype; resp.max_stride = dev->lif_cfg.max_stride; resp.max_spec = IONIC_SPEC_HIGH;
resp.udma_count = dev->lif_cfg.udma_count; resp.expdb_mask = dev->lif_cfg.expdb_mask; resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit; resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS;
if (dev->lif_cfg.sq_expdb) resp.expdb_qtypes |= IONIC_EXPDB_SQ; if (dev->lif_cfg.rq_expdb) resp.expdb_qtypes |= IONIC_EXPDB_RQ;
rc = ib_respond_udata(udata, resp); if (rc) goto err_resp;
return 0;
err_resp: rdma_user_mmap_entry_remove(ctx->mmap_phc);err_mmap_phc: rdma_user_mmap_entry_remove(ctx->mmap_dbell);err_mmap_dbell: ionic_put_dbid(dev, ctx->dbid);
return rc;}
void ionic_dealloc_ucontext(struct ib_ucontext *ibctx){ struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
rdma_user_mmap_entry_remove(ctx->mmap_phc); rdma_user_mmap_entry_remove(ctx->mmap_dbell); ionic_put_dbid(dev, ctx->dbid);}
static int ionic_mmap_phc_state(struct ionic_ibdev *dev, struct vm_area_struct *vma){ if (!(vma->vm_flags & VM_SHARED)) return -EINVAL;
if (vma->vm_flags & (VM_WRITE | VM_EXEC)) return -EPERM;
vm_flags_clear(vma, VM_MAYWRITE);
return vm_insert_page(vma, vma->vm_start, virt_to_page(dev->lif_cfg.phc_state));}
int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma){ struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); struct ionic_ctx *ctx = to_ionic_ctx(ibctx); struct rdma_user_mmap_entry *rdma_entry; struct ionic_mmap_entry *ionic_entry; int rc = 0;
rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma); if (!rdma_entry) { ibdev_dbg(&dev->ibdev, "not found %#lx\n", vma->vm_pgoff << PAGE_SHIFT); return -EINVAL; }
ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, rdma_entry);
if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) { rc = ionic_mmap_phc_state(dev, vma); rdma_user_mmap_entry_put(rdma_entry); return rc; }
ibdev_dbg(&dev->ibdev, "writecombine? %d\n", ionic_entry->mmap_flags & IONIC_MMAP_WC); if (ionic_entry->mmap_flags & IONIC_MMAP_WC) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); else vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n", vma->vm_start, ionic_entry->pfn, ionic_entry->size); rc = rdma_user_mmap_io(&ctx->ibctx, vma, ionic_entry->pfn, ionic_entry->size, vma->vm_page_prot, rdma_entry); if (rc) ibdev_dbg(&dev->ibdev, "remap failed %d\n", rc);
rdma_user_mmap_entry_put(rdma_entry); return rc;}
void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry){ struct ionic_mmap_entry *ionic_entry;
ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, rdma_entry); kfree(ionic_entry);}
int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); int rc;
rc = ib_no_udata_io(udata); if (rc) return rc;
return ionic_get_pdid(dev, &pd->pdid);}
int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); int rc;
rc = ib_no_udata_io(udata); if (rc) return rc;
ionic_put_pdid(dev, pd->pdid);
return 0;}
static int ionic_build_hdr(struct ionic_ibdev *dev, struct ib_ud_header *hdr, const struct rdma_ah_attr *attr, u16 sport, bool want_ecn){ const struct ib_global_route *grh; enum rdma_network_type net; u8 smac[ETH_ALEN]; u16 vlan; int rc;
if (attr->ah_flags != IB_AH_GRH) return -EINVAL; if (attr->type != RDMA_AH_ATTR_TYPE_ROCE) return -EINVAL;
grh = rdma_ah_read_grh(attr);
rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac); if (rc) return rc;
net = rdma_gid_attr_network_type(grh->sgid_attr);
rc = ib_ud_header_init(0, /* no payload */ 0, /* no lrh */ 1, /* yes eth */ vlan != 0xffff, 0, /* no grh */ net == RDMA_NETWORK_IPV4 ? 4 : 6, 1, /* yes udp */ 0, /* no imm */ hdr); if (rc) return rc;
ether_addr_copy(hdr->eth.smac_h, smac); ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac);
if (net == RDMA_NETWORK_IPV4) { hdr->eth.type = cpu_to_be16(ETH_P_IP); hdr->ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */ hdr->ip4.ttl = grh->hop_limit; hdr->ip4.tot_len = cpu_to_be16(0xffff); hdr->ip4.saddr = *(const __be32 *)(grh->sgid_attr->gid.raw + 12); hdr->ip4.daddr = *(const __be32 *)(grh->dgid.raw + 12);
if (want_ecn) hdr->ip4.tos = ionic_set_ecn(grh->traffic_class); else hdr->ip4.tos = ionic_clear_ecn(grh->traffic_class); } else { hdr->eth.type = cpu_to_be16(ETH_P_IPV6); hdr->grh.flow_label = cpu_to_be32(grh->flow_label); hdr->grh.hop_limit = grh->hop_limit; hdr->grh.source_gid = grh->sgid_attr->gid; hdr->grh.destination_gid = grh->dgid;
if (want_ecn) hdr->grh.traffic_class = ionic_set_ecn(grh->traffic_class); else hdr->grh.traffic_class = ionic_clear_ecn(grh->traffic_class); }
if (vlan != 0xffff) { vlan |= rdma_ah_get_sl(attr) << VLAN_PRIO_SHIFT; hdr->vlan.tag = cpu_to_be16(vlan); hdr->vlan.type = hdr->eth.type; hdr->eth.type = cpu_to_be16(ETH_P_8021Q); }
hdr->udp.sport = cpu_to_be16(sport); hdr->udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT);
return 0;}
static void ionic_set_ah_attr(struct ionic_ibdev *dev, struct rdma_ah_attr *ah_attr, struct ib_ud_header *hdr, int sgid_index){ u32 flow_label; u16 vlan = 0; u8 tos, ttl;
if (hdr->vlan_present) vlan = be16_to_cpu(hdr->vlan.tag);
if (hdr->ipv4_present) { flow_label = 0; ttl = hdr->ip4.ttl; tos = hdr->ip4.tos; *(__be16 *)(hdr->grh.destination_gid.raw + 10) = cpu_to_be16(0xffff); *(__be32 *)(hdr->grh.destination_gid.raw + 12) = hdr->ip4.daddr; } else { flow_label = be32_to_cpu(hdr->grh.flow_label); ttl = hdr->grh.hop_limit; tos = hdr->grh.traffic_class; }
memset(ah_attr, 0, sizeof(*ah_attr)); ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE; if (hdr->eth_present) ether_addr_copy(ah_attr->roce.dmac, hdr->eth.dmac_h); rdma_ah_set_sl(ah_attr, vlan >> VLAN_PRIO_SHIFT); rdma_ah_set_port_num(ah_attr, 1); rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos); rdma_ah_set_dgid_raw(ah_attr, &hdr->grh.destination_gid);}
static int ionic_create_ah_cmd(struct ionic_ibdev *dev, struct ionic_ah *ah, struct ionic_pd *pd, struct rdma_ah_attr *attr, u32 flags){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_CREATE_AH, .len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN), .cmd.create_ah = { .pd_id = cpu_to_le32(pd->pdid), .dbid_flags = cpu_to_le16(dev->lif_cfg.dbid), .id_ver = cpu_to_le32(ah->ahid), } } }; enum ionic_admin_flags admin_flags = 0; dma_addr_t hdr_dma = 0; void *hdr_buf; gfp_t gfp = GFP_ATOMIC; int rc, hdr_len = 0;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_AH) return -EBADRQC;
if (flags & RDMA_CREATE_AH_SLEEPABLE) gfp = GFP_KERNEL; else admin_flags |= IONIC_ADMIN_F_BUSYWAIT;
rc = ionic_build_hdr(dev, &ah->hdr, attr, IONIC_ROCE_UDP_SPORT, false); if (rc) return rc;
if (ah->hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) { if (ah->hdr.vlan.type == cpu_to_be16(ETH_P_IP)) wr.wqe.cmd.create_ah.csum_profile = IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP; else wr.wqe.cmd.create_ah.csum_profile = IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP; } else { if (ah->hdr.eth.type == cpu_to_be16(ETH_P_IP)) wr.wqe.cmd.create_ah.csum_profile = IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; else wr.wqe.cmd.create_ah.csum_profile = IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; }
ah->sgid_index = rdma_ah_read_grh(attr)->sgid_index;
hdr_buf = kmalloc(PAGE_SIZE, gfp); if (!hdr_buf) return -ENOMEM;
hdr_len = ib_ud_header_pack(&ah->hdr, hdr_buf); hdr_len -= IB_BTH_BYTES; hdr_len -= IB_DETH_BYTES; ibdev_dbg(&dev->ibdev, "roce packet header template\n"); print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, hdr_buf, hdr_len, true);
hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, DMA_TO_DEVICE);
rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); if (rc) goto err_dma;
wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma); wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len);
ionic_admin_post(dev, &wr); rc = ionic_admin_wait(dev, &wr, admin_flags);
dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, DMA_TO_DEVICE);err_dma: kfree(hdr_buf);
return rc;}
static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_DESTROY_AH, .len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN), .cmd.destroy_ah = { .ah_id = cpu_to_le32(ahid), }, } }; enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_AH) return -EBADRQC;
if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) admin_flags |= IONIC_ADMIN_F_BUSYWAIT;
ionic_admin_post(dev, &wr); ionic_admin_wait(dev, &wr, admin_flags);
/* No host-memory resource is associated with ah, so it is ok * to "succeed" and complete this destroy ah on the host. */ return 0;}
int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); struct rdma_ah_attr *attr = init_attr->ah_attr; struct ionic_pd *pd = to_ionic_pd(ibah->pd); struct ionic_ah *ah = to_ionic_ah(ibah); struct ionic_ah_resp resp = {}; u32 flags = init_attr->flags; int rc;
rc = ib_is_udata_in_empty(udata); if (rc) return rc;
rc = ionic_get_ahid(dev, &ah->ahid); if (rc) return rc;
rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags); if (rc) goto err_cmd;
if (udata) { resp.ahid = ah->ahid;
rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; }
return 0;
err_resp: ionic_destroy_ah_cmd(dev, ah->ahid, flags);err_cmd: ionic_put_ahid(dev, ah->ahid); return rc;}
int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr){ struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); struct ionic_ah *ah = to_ionic_ah(ibah);
ionic_set_ah_attr(dev, ah_attr, &ah->hdr, ah->sgid_index);
return 0;}
int ionic_destroy_ah(struct ib_ah *ibah, u32 flags){ struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device); struct ionic_ah *ah = to_ionic_ah(ibah); int rc;
rc = ionic_destroy_ah_cmd(dev, ah->ahid, flags); if (rc) return rc;
ionic_put_ahid(dev, ah->ahid);
return 0;}
static int ionic_create_mr_cmd(struct ionic_ibdev *dev, struct ionic_pd *pd, struct ionic_mr *mr, u64 addr, u64 length){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_CREATE_MR, .len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN), .cmd.create_mr = { .va = cpu_to_le64(addr), .length = cpu_to_le64(length), .pd_id = cpu_to_le32(pd->pdid), .page_size_log2 = mr->buf.page_size_log2, .tbl_index = cpu_to_le32(~0), .map_count = cpu_to_le32(mr->buf.tbl_pages), .dma_addr = ionic_pgtbl_dma(&mr->buf, addr), .dbid_flags = cpu_to_le16(mr->flags), .id_ver = cpu_to_le32(mr->mrid), } } }; int rc;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_MR) return -EBADRQC;
ionic_admin_post(dev, &wr); rc = ionic_admin_wait(dev, &wr, 0); if (!rc) mr->created = true;
return rc;}
static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_DESTROY_MR, .len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN), .cmd.destroy_mr = { .mr_id = cpu_to_le32(mrid), }, } };
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_MR) return -EBADRQC;
ionic_admin_post(dev, &wr);
return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);}
struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access){ struct ionic_pd *pd = to_ionic_pd(ibpd); struct ionic_mr *mr;
mr = kzalloc_obj(*mr); if (!mr) return ERR_PTR(-ENOMEM);
mr->ibmr.lkey = IONIC_DMA_LKEY; mr->ibmr.rkey = IONIC_DMA_RKEY;
if (pd) pd->flags |= IONIC_QPF_PRIVILEGED;
return &mr->ibmr;}
struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 addr, int access, struct ib_dmah *dmah, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); struct ionic_mr *mr; unsigned long pg_sz; int rc;
rc = ib_no_udata_io(udata); if (rc) return ERR_PTR(rc);
if (dmah) return ERR_PTR(-EOPNOTSUPP);
mr = kzalloc_obj(*mr); if (!mr) return ERR_PTR(-ENOMEM);
rc = ionic_get_mrid(dev, &mr->mrid); if (rc) goto err_mrid;
mr->ibmr.lkey = mr->mrid; mr->ibmr.rkey = mr->mrid; mr->ibmr.iova = addr; mr->ibmr.length = length;
mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access); if (IS_ERR(mr->umem)) { rc = PTR_ERR(mr->umem); goto err_umem; }
pg_sz = ib_umem_find_best_pgsz(mr->umem, dev->lif_cfg.page_size_supported, addr); if (!pg_sz) { rc = -EINVAL; goto err_pgtbl; }
rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); if (rc) goto err_pgtbl;
rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); if (rc) goto err_cmd;
ionic_pgtbl_unbuf(dev, &mr->buf);
return &mr->ibmr;
err_cmd: ionic_pgtbl_unbuf(dev, &mr->buf);err_pgtbl: ib_umem_release(mr->umem);err_umem: ionic_put_mrid(dev, mr->mrid);err_mrid: kfree(mr); return ERR_PTR(rc);}
struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, u64 length, u64 addr, int fd, int access, struct ib_dmah *dmah, struct uverbs_attr_bundle *attrs){ struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); struct ib_umem_dmabuf *umem_dmabuf; struct ionic_mr *mr; u64 pg_sz; int rc;
if (dmah) return ERR_PTR(-EOPNOTSUPP);
mr = kzalloc_obj(*mr); if (!mr) return ERR_PTR(-ENOMEM);
rc = ionic_get_mrid(dev, &mr->mrid); if (rc) goto err_mrid;
mr->ibmr.lkey = mr->mrid; mr->ibmr.rkey = mr->mrid; mr->ibmr.iova = addr; mr->ibmr.length = length;
mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ibdev, offset, length, fd, access); if (IS_ERR(umem_dmabuf)) { rc = PTR_ERR(umem_dmabuf); goto err_umem; }
mr->umem = &umem_dmabuf->umem;
pg_sz = ib_umem_find_best_pgsz(mr->umem, dev->lif_cfg.page_size_supported, addr); if (!pg_sz) { rc = -EINVAL; goto err_pgtbl; }
rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz); if (rc) goto err_pgtbl;
rc = ionic_create_mr_cmd(dev, pd, mr, addr, length); if (rc) goto err_cmd;
ionic_pgtbl_unbuf(dev, &mr->buf);
return &mr->ibmr;
err_cmd: ionic_pgtbl_unbuf(dev, &mr->buf);err_pgtbl: ib_umem_release(mr->umem);err_umem: ionic_put_mrid(dev, mr->mrid);err_mrid: kfree(mr); return ERR_PTR(rc);}
int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); struct ionic_mr *mr = to_ionic_mr(ibmr); int rc;
rc = ib_no_udata_io(udata); if (rc) return rc;
if (!mr->ibmr.lkey) goto out;
if (mr->created) { rc = ionic_destroy_mr_cmd(dev, mr->mrid); if (rc) return rc; }
ionic_pgtbl_unbuf(dev, &mr->buf);
if (mr->umem) ib_umem_release(mr->umem);
ionic_put_mrid(dev, mr->mrid);
out: kfree(mr);
return 0;}
struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, u32 max_sg){ struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); struct ionic_mr *mr; int rc;
if (type != IB_MR_TYPE_MEM_REG) return ERR_PTR(-EINVAL);
mr = kzalloc_obj(*mr); if (!mr) return ERR_PTR(-ENOMEM);
rc = ionic_get_mrid(dev, &mr->mrid); if (rc) goto err_mrid;
mr->ibmr.lkey = mr->mrid; mr->ibmr.rkey = mr->mrid;
mr->flags = IONIC_MRF_PHYS_MR;
rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, max_sg, PAGE_SIZE); if (rc) goto err_pgtbl;
mr->buf.tbl_pages = 0;
rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); if (rc) goto err_cmd;
return &mr->ibmr;
err_cmd: ionic_pgtbl_unbuf(dev, &mr->buf);err_pgtbl: ionic_put_mrid(dev, mr->mrid);err_mrid: kfree(mr); return ERR_PTR(rc);}
static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma){ struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); struct ionic_mr *mr = to_ionic_mr(ibmr);
ibdev_dbg(&dev->ibdev, "dma %p\n", (void *)dma); return ionic_pgtbl_page(&mr->buf, dma);}
int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset){ struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device); struct ionic_mr *mr = to_ionic_mr(ibmr); int rc;
/* mr must be allocated using ib_alloc_mr() */ if (unlikely(!mr->buf.tbl_limit)) return -EINVAL;
mr->buf.tbl_pages = 0;
if (mr->buf.tbl_buf) dma_sync_single_for_cpu(dev->lif_cfg.hwdev, mr->buf.tbl_dma, mr->buf.tbl_size, DMA_TO_DEVICE);
ibdev_dbg(&dev->ibdev, "sg %p nent %d\n", sg, sg_nents); rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page);
mr->buf.page_size_log2 = order_base_2(ibmr->page_size);
if (mr->buf.tbl_buf) dma_sync_single_for_device(dev->lif_cfg.hwdev, mr->buf.tbl_dma, mr->buf.tbl_size, DMA_TO_DEVICE);
return rc;}
int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); struct ionic_pd *pd = to_ionic_pd(ibmw->pd); struct ionic_mr *mr = to_ionic_mw(ibmw); int rc;
rc = ib_no_udata_io(udata); if (rc) return rc;
rc = ionic_get_mrid(dev, &mr->mrid); if (rc) return rc;
mr->ibmw.rkey = mr->mrid;
if (mr->ibmw.type == IB_MW_TYPE_1) mr->flags = IONIC_MRF_MW_1; else mr->flags = IONIC_MRF_MW_2;
rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0); if (rc) goto err_cmd;
return 0;
err_cmd: ionic_put_mrid(dev, mr->mrid); return rc;}
int ionic_dealloc_mw(struct ib_mw *ibmw){ struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device); struct ionic_mr *mr = to_ionic_mw(ibmw); int rc;
rc = ionic_destroy_mr_cmd(dev, mr->mrid); if (rc) return rc;
ionic_put_mrid(dev, mr->mrid);
return 0;}
static int ionic_create_cq_cmd(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_cq *cq, struct ionic_tbl_buf *buf, int udma_idx, int *out_udma_idx){ struct ionic_admin_create_cq_resp *resp_buf; const u16 dbid = ionic_ctx_dbid(dev, ctx); struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_CREATE_CQ, .len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN), .cmd.create_cq = { .eq_id = cpu_to_le32(cq->eqid), .depth_log2 = cq->q.depth_log2, .stride_log2 = cq->q.stride_log2, .page_size_log2 = buf->page_size_log2, .tbl_index = cpu_to_le32(~0), .map_count = cpu_to_le32(buf->tbl_pages), .dma_addr = ionic_pgtbl_dma(buf, 0), .dbid_flags = cpu_to_le16(dbid), .id_ver = cpu_to_le32(cq->cqid), } } }; dma_addr_t resp_buf_dma; int rc;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ) return -EBADRQC;
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { ionic_admin_post(dev, &wr); return ionic_admin_wait(dev, &wr, 0); }
resp_buf = kzalloc_obj(*resp_buf); if (!resp_buf) return -ENOMEM;
resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, sizeof(*resp_buf), DMA_FROM_DEVICE);
rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); if (rc) { rc = -ENOMEM; goto err_dma; }
wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); wr.wqe.cmd.create_cq.udma_idx = udma_idx; wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma); wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN);
ionic_admin_post(dev, &wr); rc = ionic_admin_wait(dev, &wr, 0);
dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), DMA_FROM_DEVICE); if (rc) goto err_dma;
if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) { rc = -EOPNOTSUPP; goto err_dma; }
cq->cqid = le32_to_cpu(resp_buf->id); if (out_udma_idx) *out_udma_idx = resp_buf->udma_idx;
err_dma: kfree(resp_buf);
return rc;}
static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_DESTROY_CQ, .len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN), .cmd.destroy_cq = { .cq_id = cpu_to_le32(cqid), }, } };
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_CQ) return -EBADRQC;
ionic_admin_post(dev, &wr);
return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);}
int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs){ struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); struct ib_udata *udata = &attrs->driver_udata; struct ionic_ctx *ctx = rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); struct ionic_tbl_buf buf = {}; struct ionic_cq_resp resp = {}; struct ionic_cq_req req; int udma_idx = 0, rc;
if (udata) { rc = ib_copy_validate_udata_in(udata, req, rsvd); if (rc) return rc; }
vcq->udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
if (udata) vcq->udma_mask &= req.udma_mask;
if (!vcq->udma_mask) { rc = -EINVAL; goto err_init; }
for (; udma_idx < dev->lif_cfg.udma_count; ++udma_idx) { if (!(vcq->udma_mask & BIT(udma_idx))) continue;
rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata, &req.cq[udma_idx], udma_idx); if (rc) goto err_init;
rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf, udma_idx, NULL); if (rc) goto err_cmd;
ionic_pgtbl_unbuf(dev, &buf);
rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata, &resp.cqid[udma_idx]); if (rc) goto err_post; }
vcq->ibcq.cqe = attr->cqe;
if (udata) { resp.udma_mask = vcq->udma_mask;
rc = ib_respond_udata(udata, resp); if (rc) goto err_resp; }
return 0;
err_resp: while (udma_idx) { --udma_idx; if (!(vcq->udma_mask & BIT(udma_idx))) continue; ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]);err_post: ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);err_cmd: ionic_pgtbl_unbuf(dev, &buf); ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]);err_init: ; }
return rc;}
int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); int udma_idx, rc_tmp, rc = 0;
rc = ib_no_udata_io(udata); if (rc) return rc;
for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) { --udma_idx;
if (!(vcq->udma_mask & BIT(udma_idx))) continue;
/* * Untrack the CQ before releasing its hardware ID below, so a * concurrent create that gets the same ID reused by firmware * cannot have its fresh XArray entry erased by this destroy. */ ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]);
rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); if (rc_tmp) { if (!rc) rc = rc_tmp;
continue; }
ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]); }
return rc;}
static bool pd_remote_privileged(struct ib_pd *pd){ return pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY;}
static int ionic_create_qp_cmd(struct ionic_ibdev *dev, struct ionic_pd *pd, struct ionic_cq *send_cq, struct ionic_cq *recv_cq, struct ionic_qp *qp, struct ionic_tbl_buf *sq_buf, struct ionic_tbl_buf *rq_buf, struct ib_qp_init_attr *attr, u32 ionic_flags){ const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); const u32 flags = to_ionic_qp_flags(0, 0, qp->sq_cmb & IONIC_CMB_ENABLE, qp->rq.cmb & IONIC_CMB_ENABLE, qp->sq_spec, qp->rq.spec, pd->flags & IONIC_QPF_PRIVILEGED, pd_remote_privileged(&pd->ibpd)); struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_CREATE_QP, .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN), .cmd.create_qp = { .pd_id = cpu_to_le32(pd->pdid), /* User-supplied ionic_flags are passed through to * firmware, which validates and rejects any * unsupported or unauthorized bits. */ .priv_flags = cpu_to_be32(flags | (ionic_flags & IONIC_QP_USER_FLAGS_MASK)), .type_state = to_ionic_qp_type(attr->qp_type), .dbid_flags = cpu_to_le16(dbid), .id_ver = cpu_to_le32(qp->qpid), } } };
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_QP) return -EBADRQC;
if (qp->has_sq) { wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq->cqid); wr.wqe.cmd.create_qp.sq_depth_log2 = qp->sq.depth_log2; wr.wqe.cmd.create_qp.sq_stride_log2 = qp->sq.stride_log2; wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf->page_size_log2; wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0); wr.wqe.cmd.create_qp.sq_map_count = cpu_to_le32(sq_buf->tbl_pages); wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0); }
if (!qp->srq) { wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2; wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2; wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2; wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0); wr.wqe.cmd.create_qp.rq_map_count = cpu_to_le32(rq_buf->tbl_pages); wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0); } else { wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(qp->srq->rq.qid); wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); }
ionic_admin_post(dev, &wr);
return ionic_admin_wait(dev, &wr, 0);}
static int ionic_modify_qp_cmd(struct ionic_ibdev *dev, struct ionic_pd *pd, struct ionic_qp *qp, struct ib_qp_attr *attr, int mask){ const u32 flags = to_ionic_qp_flags(attr->qp_access_flags, attr->en_sqd_async_notify, qp->sq_cmb & IONIC_CMB_ENABLE, qp->rq.cmb & IONIC_CMB_ENABLE, qp->sq_spec, qp->rq.spec, pd->flags & IONIC_QPF_PRIVILEGED, pd_remote_privileged(qp->ibqp.pd)); const u8 state = to_ionic_qp_modify_state(attr->qp_state, attr->cur_qp_state); struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_MODIFY_QP, .len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN), .cmd.mod_qp = { .attr_mask = cpu_to_be32(mask), .access_flags = cpu_to_be16(flags), .rq_psn = cpu_to_le32(attr->rq_psn), .sq_psn = cpu_to_le32(attr->sq_psn), .rate_limit_kbps = cpu_to_le32(attr->rate_limit), .pmtu = (attr->path_mtu + 7), .retry = (attr->retry_cnt | (attr->rnr_retry << 4)), .rnr_timer = attr->min_rnr_timer, .retry_timeout = attr->timeout, .type_state = state, .id_ver = cpu_to_le32(qp->qpid), } } }; const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); void *hdr_buf = NULL; dma_addr_t hdr_dma = 0; int rc, hdr_len = 0; u16 sport;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_QP) return -EBADRQC;
if ((mask & IB_QP_MAX_DEST_RD_ATOMIC) && attr->max_dest_rd_atomic) { /* Note, round up/down was already done for allocating * resources on the device. The allocation order is in cache * line size. We can't use the order of the resource * allocation to determine the order wqes here, because for * queue length <= one cache line it is not distinct. * * Therefore, order wqes is computed again here. * * Account for hole and round up to the next order. */ wr.wqe.cmd.mod_qp.rsq_depth = order_base_2(attr->max_dest_rd_atomic + 1); wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0); }
if ((mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { /* Account for hole and round down to the next order */ wr.wqe.cmd.mod_qp.rrq_depth = order_base_2(attr->max_rd_atomic + 2) - 1; wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0); }
if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->dest_qp_num); else wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->qkey);
if (mask & IB_QP_AV) { if (!qp->hdr) return -ENOMEM;
sport = rdma_get_udp_sport(grh->flow_label, qp->qpid, attr->dest_qp_num);
rc = ionic_build_hdr(dev, qp->hdr, &attr->ah_attr, sport, true); if (rc) return rc;
qp->sgid_index = grh->sgid_index;
hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!hdr_buf) return -ENOMEM;
hdr_len = ib_ud_header_pack(qp->hdr, hdr_buf); hdr_len -= IB_BTH_BYTES; hdr_len -= IB_DETH_BYTES; ibdev_dbg(&dev->ibdev, "roce packet header template\n"); print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1, hdr_buf, hdr_len, true);
hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len, DMA_TO_DEVICE);
rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); if (rc) goto err_dma;
if (qp->hdr->ipv4_present) { wr.wqe.cmd.mod_qp.tfp_csum_profile = qp->hdr->vlan_present ? IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP : IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP; } else { wr.wqe.cmd.mod_qp.tfp_csum_profile = qp->hdr->vlan_present ? IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP : IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP; }
wr.wqe.cmd.mod_qp.ah_id_len = cpu_to_le32(qp->ahid | (hdr_len << 24)); wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma);
wr.wqe.cmd.mod_qp.en_pcp = attr->ah_attr.sl; wr.wqe.cmd.mod_qp.ip_dscp = grh->traffic_class >> 2; }
ionic_admin_post(dev, &wr);
rc = ionic_admin_wait(dev, &wr, 0);
if (mask & IB_QP_AV) dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len, DMA_TO_DEVICE);err_dma: if (mask & IB_QP_AV) kfree(hdr_buf);
return rc;}
static int ionic_query_qp_cmd(struct ionic_ibdev *dev, struct ionic_qp *qp, struct ib_qp_attr *attr, int mask){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_QUERY_QP, .len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN), .cmd.query_qp = { .id_ver = cpu_to_le32(qp->qpid), }, } }; struct ionic_v1_admin_query_qp_sq *query_sqbuf; struct ionic_v1_admin_query_qp_rq *query_rqbuf; dma_addr_t query_sqdma; dma_addr_t query_rqdma; dma_addr_t hdr_dma = 0; void *hdr_buf = NULL; int flags, rc;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_QUERY_QP) return -EBADRQC;
if (qp->has_sq) { bool expdb = !!(qp->sq_cmb & IONIC_CMB_EXPDB);
attr->cap.max_send_sge = ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, qp->sq_spec, expdb); attr->cap.max_inline_data = ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb); }
if (!qp->srq) { attr->cap.max_recv_sge = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec, qp->rq.cmb & IONIC_CMB_EXPDB); }
query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!query_sqbuf) return -ENOMEM;
query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!query_rqbuf) { rc = -ENOMEM; goto err_rqbuf; }
query_sqdma = dma_map_single(dev->lif_cfg.hwdev, query_sqbuf, PAGE_SIZE, DMA_FROM_DEVICE); rc = dma_mapping_error(dev->lif_cfg.hwdev, query_sqdma); if (rc) goto err_sqdma;
query_rqdma = dma_map_single(dev->lif_cfg.hwdev, query_rqbuf, PAGE_SIZE, DMA_FROM_DEVICE); rc = dma_mapping_error(dev->lif_cfg.hwdev, query_rqdma); if (rc) goto err_rqdma;
if (mask & IB_QP_AV) { hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!hdr_buf) { rc = -ENOMEM; goto err_hdrbuf; }
hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, PAGE_SIZE, DMA_FROM_DEVICE); rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); if (rc) goto err_hdrdma; }
wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma); wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma); wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma); wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp->ahid);
ionic_admin_post(dev, &wr);
rc = ionic_admin_wait(dev, &wr, 0);
if (rc) goto err_hdrdma;
flags = be16_to_cpu(query_sqbuf->access_perms_flags | query_rqbuf->access_perms_flags);
print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1, query_sqbuf, sizeof(*query_sqbuf), true); print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1, query_rqbuf, sizeof(*query_rqbuf), true); ibdev_dbg(&dev->ibdev, "query qp %u state_pmtu %#x flags %#x", qp->qpid, query_rqbuf->state_pmtu, flags);
attr->qp_state = from_ionic_qp_state(query_rqbuf->state_pmtu >> 4); attr->cur_qp_state = attr->qp_state; attr->path_mtu = (query_rqbuf->state_pmtu & 0xf) - 7; attr->path_mig_state = IB_MIG_MIGRATED; attr->qkey = be32_to_cpu(query_sqbuf->qkey_dest_qpn); attr->rq_psn = be32_to_cpu(query_sqbuf->rq_psn); attr->sq_psn = be32_to_cpu(query_rqbuf->sq_psn); attr->dest_qp_num = attr->qkey; attr->qp_access_flags = from_ionic_qp_flags(flags); attr->pkey_index = 0; attr->alt_pkey_index = 0; attr->en_sqd_async_notify = !!(flags & IONIC_QPF_SQD_NOTIFY); attr->sq_draining = !!(flags & IONIC_QPF_SQ_DRAINING); attr->max_rd_atomic = BIT(query_rqbuf->rrq_depth) - 1; attr->max_dest_rd_atomic = BIT(query_rqbuf->rsq_depth) - 1; attr->min_rnr_timer = query_sqbuf->rnr_timer; attr->port_num = 0; attr->timeout = query_sqbuf->retry_timeout; attr->retry_cnt = query_rqbuf->retry_rnrtry & 0xf; attr->rnr_retry = query_rqbuf->retry_rnrtry >> 4; attr->alt_port_num = 0; attr->alt_timeout = 0; attr->rate_limit = be32_to_cpu(query_sqbuf->rate_limit_kbps);
if (mask & IB_QP_AV) ionic_set_ah_attr(dev, &attr->ah_attr, qp->hdr, qp->sgid_index);
err_hdrdma: if (mask & IB_QP_AV) { dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE); kfree(hdr_buf); }err_hdrbuf: dma_unmap_single(dev->lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf), DMA_FROM_DEVICE);err_rqdma: dma_unmap_single(dev->lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf), DMA_FROM_DEVICE);err_sqdma: kfree(query_rqbuf);err_rqbuf: kfree(query_sqbuf);
return rc;}
static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_DESTROY_QP, .len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN), .cmd.destroy_qp = { .qp_id = cpu_to_le32(qpid), }, } };
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_QP) return -EBADRQC;
ionic_admin_post(dev, &wr);
return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);}
static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev, uint32_t wqe_size){ switch (wqe_size) { case 64: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_64; case 128: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_128; case 256: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_256; case 512: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_512; }
return false;}
static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev, struct ionic_qp *qp, struct ib_udata *udata, int max_data){ u8 expdb_stride_log2 = 0; bool expdb; int rc;
if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) goto not_in_cmb;
if (qp->sq_cmb & ~IONIC_CMB_SUPPORTED) { if (qp->sq_cmb & IONIC_CMB_REQUIRE) goto not_in_cmb;
qp->sq_cmb &= IONIC_CMB_SUPPORTED; }
if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.sq_expdb) { if (qp->sq_cmb & IONIC_CMB_REQUIRE) goto not_in_cmb;
qp->sq_cmb &= ~IONIC_CMB_EXPDB; }
qp->sq_cmb_order = order_base_2(qp->sq.size / PAGE_SIZE);
if (qp->sq_cmb_order >= IONIC_SQCMB_ORDER) goto not_in_cmb;
if (qp->sq_cmb & IONIC_CMB_EXPDB) expdb_stride_log2 = qp->sq.stride_log2;
rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->sq_cmb_pgid, &qp->sq_cmb_addr, qp->sq_cmb_order, expdb_stride_log2, &expdb); if (rc) goto not_in_cmb;
if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !expdb) { if (qp->sq_cmb & IONIC_CMB_REQUIRE) goto err_map;
qp->sq_cmb &= ~IONIC_CMB_EXPDB; }
return;
err_map: ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order);not_in_cmb: if (qp->sq_cmb & IONIC_CMB_REQUIRE) ibdev_dbg(&dev->ibdev, "could not place sq in cmb as required\n");
qp->sq_cmb = 0; qp->sq_cmb_order = IONIC_RES_INVALID; qp->sq_cmb_pgid = 0; qp->sq_cmb_addr = 0;}
static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_qp *qp){ if (!(qp->sq_cmb & IONIC_CMB_ENABLE)) return;
if (ctx) rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order);}
static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_qp *qp, struct ionic_qdesc *sq, struct ionic_tbl_buf *buf, int max_wr, int max_sge, int max_data, int sq_spec, struct ib_udata *udata){ u32 wqe_size; int rc = 0;
qp->sq_msn_prod = 0; qp->sq_msn_cons = 0;
if (!qp->has_sq) { if (buf) { buf->tbl_buf = NULL; buf->tbl_limit = 0; buf->tbl_pages = 0; } if (udata) rc = ionic_validate_qdesc_zero(sq);
return rc; }
rc = -EINVAL;
if (max_wr < 0 || max_wr > 0xffff) return rc;
if (max_sge < 1) return rc;
if (max_sge > min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0, qp->sq_cmb & IONIC_CMB_EXPDB), IONIC_SPEC_HIGH)) return rc;
if (max_data < 0) return rc;
if (max_data > ionic_v1_send_wqe_max_data(dev->lif_cfg.max_stride, qp->sq_cmb & IONIC_CMB_EXPDB)) return rc;
if (udata) { rc = ionic_validate_qdesc(sq); if (rc) return rc;
qp->sq_spec = sq_spec;
qp->sq.ptr = NULL; qp->sq.size = sq->size; qp->sq.mask = sq->mask; qp->sq.depth_log2 = sq->depth_log2; qp->sq.stride_log2 = sq->stride_log2;
qp->sq_meta = NULL; qp->sq_msn_idx = NULL;
qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0); if (IS_ERR(qp->sq_umem)) return PTR_ERR(qp->sq_umem); } else { qp->sq_umem = NULL;
qp->sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec); if (sq_spec && !qp->sq_spec) ibdev_dbg(&dev->ibdev, "init sq: max_sge %u disables spec\n", max_sge);
if (qp->sq_cmb & IONIC_CMB_EXPDB) { wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, qp->sq_spec, true);
if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) qp->sq_cmb &= ~IONIC_CMB_EXPDB; }
if (!(qp->sq_cmb & IONIC_CMB_EXPDB)) wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data, qp->sq_spec, false);
rc = ionic_queue_init(&qp->sq, dev->lif_cfg.hwdev, max_wr, wqe_size); if (rc) return rc;
ionic_queue_dbell_init(&qp->sq, qp->qpid);
qp->sq_meta = kmalloc_objs(*qp->sq_meta, (u32)qp->sq.mask + 1); if (!qp->sq_meta) { rc = -ENOMEM; goto err_sq_meta; }
qp->sq_msn_idx = kmalloc_array((u32)qp->sq.mask + 1, sizeof(*qp->sq_msn_idx), GFP_KERNEL); if (!qp->sq_msn_idx) { rc = -ENOMEM; goto err_sq_msn; } }
ionic_qp_sq_init_cmb(dev, qp, udata, max_data);
if (qp->sq_cmb & IONIC_CMB_ENABLE) rc = ionic_pgtbl_init(dev, buf, NULL, (u64)qp->sq_cmb_pgid << PAGE_SHIFT, 1, PAGE_SIZE); else rc = ionic_pgtbl_init(dev, buf, qp->sq_umem, qp->sq.dma, 1, PAGE_SIZE); if (rc) goto err_sq_tbl;
return 0;
err_sq_tbl: ionic_qp_sq_destroy_cmb(dev, ctx, qp); kfree(qp->sq_msn_idx);err_sq_msn: kfree(qp->sq_meta);err_sq_meta: if (qp->sq_umem) ib_umem_release(qp->sq_umem); else ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); return rc;}
static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_qp *qp){ if (!qp->has_sq) return;
ionic_qp_sq_destroy_cmb(dev, ctx, qp);
kfree(qp->sq_msn_idx); kfree(qp->sq_meta);
if (qp->sq_umem) ib_umem_release(qp->sq_umem); else ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev);}
static void ionic_rq_mmap_cmb(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_rq *rq, u64 *cmb_offset){ bool wc;
/* set mapping by default to uncached for * expdb (to guarantee writes order) otherwise * writecombine, unless this default is * overridden by userspace */ if ((rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == (IONIC_CMB_WC | IONIC_CMB_UC)) { ibdev_dbg(&dev->ibdev, "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC set, using default driver mapping\n"); rq->cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); }
if (rq->cmb & IONIC_CMB_EXPDB) wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC; else wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC;
/* let userspace know the mapping */ if (wc) rq->cmb |= IONIC_CMB_WC; else rq->cmb |= IONIC_CMB_UC;
rq->mmap_cmb = ionic_mmap_entry_insert(ctx, rq->q.size, PHYS_PFN(rq->cmb_addr), wc ? IONIC_MMAP_WC : 0, cmb_offset);}
static void ionic_rq_init_cmb(struct ionic_ibdev *dev, struct ionic_rq *rq, struct ib_udata *udata){ u8 expdb_stride_log2 = 0; bool expdb; int rc;
if (!(rq->cmb & IONIC_CMB_ENABLE)) goto not_in_cmb;
if (rq->cmb & ~IONIC_CMB_SUPPORTED) { if (rq->cmb & IONIC_CMB_REQUIRE) goto not_in_cmb;
rq->cmb &= IONIC_CMB_SUPPORTED; }
if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { if (rq->cmb & IONIC_CMB_REQUIRE) goto not_in_cmb;
rq->cmb &= ~IONIC_CMB_EXPDB; }
rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE);
if (rq->cmb_order >= IONIC_RQCMB_ORDER) goto not_in_cmb;
if (rq->cmb & IONIC_CMB_EXPDB) expdb_stride_log2 = rq->q.stride_log2;
rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid, &rq->cmb_addr, rq->cmb_order, expdb_stride_log2, &expdb); if (rc) goto not_in_cmb;
if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) { if (rq->cmb & IONIC_CMB_REQUIRE) goto err_map;
rq->cmb &= ~IONIC_CMB_EXPDB; }
return;
err_map: ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order);not_in_cmb: if (rq->cmb & IONIC_CMB_REQUIRE) ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n");
rq->cmb = 0; rq->cmb_order = IONIC_RES_INVALID; rq->cmb_pgid = 0; rq->cmb_addr = 0;}
static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_rq *rq){ if (!(rq->cmb & IONIC_CMB_ENABLE)) return;
if (ctx) rdma_user_mmap_entry_remove(rq->mmap_cmb);
ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order);}
static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_rq *rq, struct ionic_qdesc *qdesc, struct ionic_tbl_buf *buf, int max_wr, int max_sge, int rq_spec, struct ib_udata *udata){ int rc = 0, i; u32 wqe_size;
rc = -EINVAL;
if (max_wr < 0 || max_wr > 0xffff) return rc;
if (max_sge < 1) return rc;
if (max_sge > min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), IONIC_SPEC_HIGH)) return rc;
if (udata) { rc = ionic_validate_qdesc(qdesc); if (rc) return rc;
rq->spec = rq_spec;
rq->q.ptr = NULL; rq->q.size = qdesc->size; rq->q.mask = qdesc->mask; rq->q.depth_log2 = qdesc->depth_log2; rq->q.stride_log2 = qdesc->stride_log2;
rq->meta = NULL;
rq->umem = ib_umem_get_va(&dev->ibdev, qdesc->addr, qdesc->size, 0); if (IS_ERR(rq->umem)) return PTR_ERR(rq->umem); } else { rq->umem = NULL;
rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec); if (rq_spec && !rq->spec) ibdev_dbg(&dev->ibdev, "init rq: max_sge %u disables spec\n", max_sge);
if (rq->cmb & IONIC_CMB_EXPDB) { wqe_size = ionic_v1_recv_wqe_min_size(max_sge, rq->spec, true);
if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) rq->cmb &= ~IONIC_CMB_EXPDB; }
if (!(rq->cmb & IONIC_CMB_EXPDB)) wqe_size = ionic_v1_recv_wqe_min_size(max_sge, rq->spec, false);
rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev, max_wr, wqe_size); if (rc) return rc;
ionic_queue_dbell_init(&rq->q, rq->qid);
rq->meta = kmalloc_objs(*rq->meta, (u32)rq->q.mask + 1); if (!rq->meta) { rc = -ENOMEM; goto err_rq_meta; }
for (i = 0; i < rq->q.mask; ++i) rq->meta[i].next = &rq->meta[i + 1]; rq->meta[i].next = IONIC_META_LAST; rq->meta_head = &rq->meta[0]; }
ionic_rq_init_cmb(dev, rq, udata);
if (rq->cmb & IONIC_CMB_ENABLE) rc = ionic_pgtbl_init(dev, buf, NULL, (u64)rq->cmb_pgid << PAGE_SHIFT, 1, PAGE_SIZE); else rc = ionic_pgtbl_init(dev, buf, rq->umem, rq->q.dma, 1, PAGE_SIZE); if (rc) goto err_rq_tbl;
return 0;
err_rq_tbl: ionic_rq_destroy_cmb(dev, ctx, rq); kfree(rq->meta);err_rq_meta: if (rq->umem) ib_umem_release(rq->umem); else ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); return rc;}
static void ionic_rq_destroy(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_rq *rq){ ionic_rq_destroy_cmb(dev, ctx, rq);
kfree(rq->meta);
if (rq->umem) ib_umem_release(rq->umem); else ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev);}
int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); struct ionic_tbl_buf sq_buf = {}, rq_buf = {}; struct ionic_pd *pd = to_ionic_pd(ibqp->pd); struct ionic_qp *qp = to_ionic_qp(ibqp); struct ionic_ctx *ctx = rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_qp_resp resp = {}; struct ionic_qp_req req = {}; struct ionic_cq *cq; u8 udma_mask; void *entry; int rc;
if (udata) { rc = ib_copy_validate_udata_in(udata, req, rsvd); if (rc) return rc;
if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK) return -EINVAL;
if (req.rsvd_pad) return -EINVAL; } else { req.sq_spec = IONIC_SPEC_HIGH; req.rq_spec = IONIC_SPEC_HIGH; }
if (attr->qp_type == IB_QPT_SMI || attr->qp_type > IB_QPT_UD) return -EOPNOTSUPP;
qp->state = IB_QPS_RESET;
INIT_LIST_HEAD(&qp->cq_poll_sq); INIT_LIST_HEAD(&qp->cq_flush_sq); INIT_LIST_HEAD(&qp->rq.cq_flush);
spin_lock_init(&qp->sq_lock); spin_lock_init(&qp->rq.lock);
qp->has_sq = 1;
if (attr->srq) qp->srq = to_ionic_srq(attr->srq); else qp->srq = NULL;
if (attr->qp_type == IB_QPT_GSI) { rc = ionic_get_gsi_qpid(dev, &qp->qpid); } else { udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
if (attr->send_cq) udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask;
if (attr->recv_cq) udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask;
if (attr->srq) udma_mask &= BIT(to_ionic_srq(attr->srq)->udma_idx);
if (udata && req.udma_mask) udma_mask &= req.udma_mask;
if (!udma_mask) return -EINVAL;
rc = ionic_get_qpid(dev, &qp->qpid, &qp->udma_idx, udma_mask); } if (rc) return rc;
qp->sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR; qp->has_ah = attr->qp_type == IB_QPT_RC;
if (qp->has_ah) { qp->hdr = kzalloc_obj(*qp->hdr); if (!qp->hdr) { rc = -ENOMEM; goto err_ah_alloc; }
rc = ionic_get_ahid(dev, &qp->ahid); if (rc) goto err_ahid; }
if (udata && (req.sq_cmb & IONIC_CMB_ENABLE)) qp->sq_cmb = req.sq_cmb;
rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf, attr->cap.max_send_wr, attr->cap.max_send_sge, attr->cap.max_inline_data, req.sq_spec, udata); if (rc) goto err_sq;
if (qp->srq) { rq_buf.tbl_buf = NULL; rq_buf.tbl_limit = 0; rq_buf.tbl_pages = 0;
if (udata) rc = ionic_validate_qdesc_zero(&req.rq); } else { if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) qp->rq.cmb = req.rq_cmb;
/* for non-srq qps, rq qid is same as qpid */ qp->rq.qid = qp->qpid; rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, attr->cap.max_recv_wr, attr->cap.max_recv_sge, req.rq_spec, udata); } if (rc) goto err_rq;
rc = ionic_create_qp_cmd(dev, pd, to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx), qp, &sq_buf, &rq_buf, attr, req.ionic_flags); if (rc) goto err_cmd;
if (udata) { resp.qpid = qp->qpid; resp.udma_idx = qp->udma_idx;
if (qp->sq_cmb & IONIC_CMB_ENABLE) { bool wc;
if ((qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == (IONIC_CMB_WC | IONIC_CMB_UC)) { ibdev_dbg(&dev->ibdev, "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); qp->sq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); }
wc = (qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC;
/* let userspace know the mapping */ if (wc) qp->sq_cmb |= IONIC_CMB_WC; else qp->sq_cmb |= IONIC_CMB_UC;
qp->mmap_sq_cmb = ionic_mmap_entry_insert(ctx, qp->sq.size, PHYS_PFN(qp->sq_cmb_addr), wc ? IONIC_MMAP_WC : 0, &resp.sq_cmb_offset); if (!qp->mmap_sq_cmb) { rc = -ENOMEM; goto err_qp_cmd; }
resp.sq_cmb = qp->sq_cmb; }
if (!qp->srq && (qp->rq.cmb & IONIC_CMB_ENABLE)) { ionic_rq_mmap_cmb(dev, ctx, &qp->rq, &resp.rq_cmb_offset);
if (!qp->rq.mmap_cmb) { rc = -ENOMEM; goto err_qp_cmd; }
resp.rq_cmb = qp->rq.cmb; }
rc = ib_respond_udata(udata, resp); if (rc) goto err_qp_cmd; }
ionic_pgtbl_unbuf(dev, &rq_buf); ionic_pgtbl_unbuf(dev, &sq_buf);
qp->ibqp.qp_num = qp->qpid;
init_completion(&qp->qp_rel_comp); kref_init(&qp->qp_kref);
entry = xa_store_irq(&dev->qp_tbl, qp->qpid, qp, GFP_KERNEL); if (entry) { if (!xa_is_err(entry)) rc = -EINVAL; else rc = xa_err(entry);
goto err_qp_cmd; }
if (qp->has_sq) { cq = to_ionic_vcq_cq(attr->send_cq, qp->udma_idx);
attr->cap.max_send_wr = qp->sq.mask; attr->cap.max_send_sge = ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, qp->sq_spec, qp->sq_cmb & IONIC_CMB_EXPDB); attr->cap.max_inline_data = ionic_v1_send_wqe_max_data(qp->sq.stride_log2, qp->sq_cmb & IONIC_CMB_EXPDB); qp->sq_cqid = cq->cqid; }
if (!qp->srq) { cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx);
attr->cap.max_recv_wr = qp->rq.q.mask; attr->cap.max_recv_sge = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec, qp->rq.cmb & IONIC_CMB_EXPDB); qp->rq.cqid = cq->cqid; }
return 0;
err_qp_cmd: ionic_destroy_qp_cmd(dev, qp->qpid);err_cmd: if (!qp->srq) { ionic_pgtbl_unbuf(dev, &rq_buf); ionic_rq_destroy(dev, ctx, &qp->rq); }err_rq: ionic_pgtbl_unbuf(dev, &sq_buf); ionic_qp_sq_destroy(dev, ctx, qp);err_sq: if (qp->has_ah) ionic_put_ahid(dev, qp->ahid);err_ahid: kfree(qp->hdr);err_ah_alloc: ionic_put_qpid(dev, qp->qpid); return rc;}
void ionic_notify_flush_cq(struct ionic_cq *cq){ if (cq->flush && cq->vcq->ibcq.comp_handler) cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, cq->vcq->ibcq.cq_context);}
static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp){ if (qp->ibqp.send_cq) ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx)); if (qp->ibqp.recv_cq && qp->ibqp.recv_cq != qp->ibqp.send_cq) ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx));}
void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp){ unsigned long irqflags; struct ionic_cq *cq;
if (qp->ibqp.send_cq) { cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
/* Hold the CQ lock and QP sq_lock to set up flush */ spin_lock_irqsave(&cq->lock, irqflags); spin_lock(&qp->sq_lock); qp->sq_flush = true; if (!ionic_queue_empty(&qp->sq)) { cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); } spin_unlock(&qp->sq_lock); spin_unlock_irqrestore(&cq->lock, irqflags); }
if (qp->ibqp.recv_cq && !qp->srq) { cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
/* Hold the CQ lock and QP rq.lock to set up flush */ spin_lock_irqsave(&cq->lock, irqflags); spin_lock(&qp->rq.lock); qp->rq.flush = true; if (!ionic_queue_empty(&qp->rq.q)) { cq->flush = true; list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); } spin_unlock(&qp->rq.lock); spin_unlock_irqrestore(&cq->lock, irqflags); }}
static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid){ struct ionic_v1_cqe *qcqe; int prod, qtf, qid, type; bool color;
if (!cq->q.ptr) return;
color = cq->color; prod = cq->q.prod; qcqe = ionic_queue_at(&cq->q, prod);
while (color == ionic_v1_cqe_color(qcqe)) { qtf = ionic_v1_cqe_qtf(qcqe); qid = ionic_v1_cqe_qtf_qid(qtf); type = ionic_v1_cqe_qtf_type(qtf);
if (qid == qpid && type != IONIC_V1_CQE_TYPE_ADMIN) ionic_v1_cqe_clean(qcqe);
prod = ionic_queue_next(&cq->q, prod); qcqe = ionic_queue_at(&cq->q, prod); color = ionic_color_wrap(prod, color); }}
static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp){ unsigned long irqflags; struct ionic_cq *cq; int i;
local_irq_save(irqflags);
if (qp->ibqp.send_cq) { cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); spin_lock(&cq->lock); ionic_clean_cq(cq, qp->qpid); spin_unlock(&cq->lock); }
if (qp->ibqp.recv_cq) { cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); spin_lock(&cq->lock); ionic_clean_cq(cq, qp->qpid); spin_unlock(&cq->lock); }
if (qp->has_sq) { spin_lock(&qp->sq_lock); qp->sq_flush = false; qp->sq_flush_rcvd = false; qp->sq_msn_prod = 0; qp->sq_msn_cons = 0; qp->sq.prod = 0; qp->sq.cons = 0; spin_unlock(&qp->sq_lock); }
if (!qp->srq) { spin_lock(&qp->rq.lock); qp->rq.flush = false; qp->rq.q.prod = 0; qp->rq.q.cons = 0; if (qp->rq.meta) { for (i = 0; i < qp->rq.q.mask; ++i) qp->rq.meta[i].next = &qp->rq.meta[i + 1]; qp->rq.meta[i].next = IONIC_META_LAST; } qp->rq.meta_head = &qp->rq.meta[0]; spin_unlock(&qp->rq.lock); }
local_irq_restore(irqflags);}
static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state, enum ib_qp_state attr_state){ if (q_state == attr_state) return true;
if (attr_state == IB_QPS_ERR) return true;
if (attr_state == IB_QPS_SQE) return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD;
return false;}
static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask, enum ib_qp_type type, enum ib_qp_state cur, enum ib_qp_state next){ if (!ext_mask) return true;
if (ext_mask & ~IB_QP_RATE_LIMIT) return false;
/* Rate limit is only supported for RC QPs during specific transitions */ return type == IB_QPT_RC && ((cur == IB_QPS_INIT && next == IB_QPS_RTR) || (cur == IB_QPS_RTR && next == IB_QPS_RTS) || (cur == IB_QPS_RTS && next == IB_QPS_RTS));}
static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr, int mask){ enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ? attr->cur_qp_state : qp->state; enum ib_qp_state next_state = (mask & IB_QP_STATE) ? attr->qp_state : cur_state;
if ((mask & IB_QP_CUR_STATE) && !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state)) return -EINVAL;
if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) || !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS, qp->ibqp.qp_type, cur_state, next_state)) return -EINVAL;
/* unprivileged qp not allowed privileged qkey */ if ((mask & IB_QP_QKEY) && (attr->qkey & 0x80000000) && qp->ibqp.uobject) return -EPERM;
return 0;}
int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); struct ionic_pd *pd = to_ionic_pd(ibqp->pd); struct ionic_qp *qp = to_ionic_qp(ibqp); int rc;
rc = ib_no_udata_io(udata); if (rc) return rc;
rc = ionic_check_modify_qp(qp, attr, mask); if (rc) return rc;
if (mask & IB_QP_CAP) return -EINVAL;
rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask); if (rc) return rc;
if (mask & IB_QP_STATE) { qp->state = attr->qp_state;
if (attr->qp_state == IB_QPS_ERR) { ionic_flush_qp(dev, qp); ionic_notify_qp_cqs(dev, qp); } else if (attr->qp_state == IB_QPS_RESET) { ionic_reset_qp(dev, qp); } }
return 0;}
int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_qp_init_attr *init_attr){ struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); struct ionic_qp *qp = to_ionic_qp(ibqp); int rc;
memset(attr, 0, sizeof(*attr)); memset(init_attr, 0, sizeof(*init_attr));
rc = ionic_query_qp_cmd(dev, qp, attr, mask); if (rc) return rc;
if (qp->has_sq) attr->cap.max_send_wr = qp->sq.mask;
if (!qp->srq) attr->cap.max_recv_wr = qp->rq.q.mask;
init_attr->event_handler = ibqp->event_handler; init_attr->qp_context = ibqp->qp_context; init_attr->send_cq = ibqp->send_cq; init_attr->recv_cq = ibqp->recv_cq; init_attr->srq = ibqp->srq; init_attr->xrcd = ibqp->xrcd; init_attr->cap = attr->cap; init_attr->sq_sig_type = qp->sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; init_attr->qp_type = ibqp->qp_type; init_attr->create_flags = 0; init_attr->port_num = 0; init_attr->rwq_ind_tbl = ibqp->rwq_ind_tbl; init_attr->source_qpn = 0;
return rc;}
int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata){ struct ionic_ctx *ctx = rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); struct ionic_qp *qp = to_ionic_qp(ibqp); unsigned long irqflags; struct ionic_cq *cq; int rc;
rc = ib_no_udata_io(udata); if (rc) return rc;
rc = ionic_destroy_qp_cmd(dev, qp->qpid); if (rc) return rc;
xa_erase_irq(&dev->qp_tbl, qp->qpid);
kref_put(&qp->qp_kref, ionic_qp_complete); wait_for_completion(&qp->qp_rel_comp);
if (qp->ibqp.send_cq) { cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx); spin_lock_irqsave(&cq->lock, irqflags); ionic_clean_cq(cq, qp->qpid); list_del(&qp->cq_poll_sq); list_del(&qp->cq_flush_sq); spin_unlock_irqrestore(&cq->lock, irqflags); }
if (qp->ibqp.recv_cq) { cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); spin_lock_irqsave(&cq->lock, irqflags); ionic_clean_cq(cq, qp->qpid); list_del(&qp->rq.cq_flush); spin_unlock_irqrestore(&cq->lock, irqflags); }
if (!qp->srq) ionic_rq_destroy(dev, ctx, &qp->rq);
ionic_qp_sq_destroy(dev, ctx, qp); if (qp->has_ah) { ionic_put_ahid(dev, qp->ahid); kfree(qp->hdr); } ionic_put_qpid(dev, qp->qpid);
return 0;}
static int ionic_create_srq_cmd(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_srq *srq, struct ionic_pd *pd, struct ionic_tbl_buf *buf, u8 udma_mask){ const u16 flags = (srq->rq.cmb & IONIC_CMB_ENABLE) ? IONIC_SRQF_CMB : 0; struct ionic_admin_create_srq_resp *resp_buf; const u16 dbid = ionic_ctx_dbid(dev, ctx); struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_CREATE_SRQ, .len = cpu_to_le16(IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN), .cmd.create_srq = { .pd_id = cpu_to_le32(pd->pdid), .depth_log2 = srq->rq.q.depth_log2, .stride_log2 = srq->rq.q.stride_log2, .page_size_log2 = buf->page_size_log2, .map_count = cpu_to_le32(buf->tbl_pages), .dma_addr = ionic_pgtbl_dma(buf, 0), .dbid = cpu_to_le16(dbid), .qid = cpu_to_le32(srq->rq.qid), .low_wqes_limit = cpu_to_le16(srq->srq_limit), .udma_mask = udma_mask, .flags = cpu_to_le16(flags), } } }; dma_addr_t resp_buf_dma; int rc;
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_SRQ) return -EOPNOTSUPP;
resp_buf = kzalloc_obj(*resp_buf); if (!resp_buf) return -ENOMEM;
resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, sizeof(*resp_buf), DMA_FROM_DEVICE);
rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); if (rc) goto err_dma;
wr.wqe.cmd.create_srq.resp_dma_addr = cpu_to_le64(resp_buf_dma); wr.wqe.cmd.create_srq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN);
ionic_admin_post(dev, &wr);
rc = ionic_admin_wait(dev, &wr, 0);
dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), DMA_FROM_DEVICE); if (rc) goto err_dma;
if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN) { rc = -EOPNOTSUPP; goto err_dma; }
srq->rq.qid = le32_to_cpu(resp_buf->id); srq->udma_idx = resp_buf->udma_idx;
err_dma: kfree(resp_buf);
return rc;}
static int ionic_destroy_srq_cmd(struct ionic_ibdev *dev, u32 srqid){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_DESTROY_SRQ, .len = cpu_to_le16(IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN), .cmd.destroy_srq = { .qid = cpu_to_le32(srqid), }, } };
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_SRQ) return -EOPNOTSUPP;
ionic_admin_post(dev, &wr);
return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);}
static int ionic_modify_srq_cmd(struct ionic_ibdev *dev, u32 srqid, u16 srq_limit){ struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), .wqe = { .op = IONIC_V1_ADMIN_MODIFY_SRQ, .len = cpu_to_le16(IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN), .cmd.modify_srq = { .qid = cpu_to_le32(srqid), .low_wqes_limit = cpu_to_le16(srq_limit), }, } };
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_SRQ) return -EOPNOTSUPP;
ionic_admin_post(dev, &wr);
return ionic_admin_wait(dev, &wr, 0);}
int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); struct ionic_pd *pd = to_ionic_pd(ibsrq->pd); struct ionic_srq *srq = to_ionic_srq(ibsrq); struct ionic_ctx *ctx = rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_srq_resp resp = {}; struct ionic_srq_req req = {}; struct ionic_tbl_buf buf = {}; u8 udma_mask; int rc;
if (!ctx) return -EOPNOTSUPP;
if (attr->srq_type != IB_SRQT_BASIC) return -EOPNOTSUPP;
if (attr->attr.max_sge > IONIC_MAX_SRQ_SGES) return -EINVAL;
if (attr->attr.srq_limit >= IONIC_MAX_SRQ_LIMIT) return -EINVAL;
udma_mask = BIT(dev->lif_cfg.udma_count) - 1; if (udata) { rc = ib_copy_validate_udata_in(udata, req, rsvd); if (rc) return rc;
if (memchr_inv(req.rsvd, 0, sizeof(req.rsvd))) return -EINVAL;
udma_mask &= req.udma_mask; }
if (!udma_mask) return -EINVAL;
if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) srq->rq.cmb = req.rq_cmb;
rc = ionic_rq_init(dev, ctx, &srq->rq, &req.rq, &buf, attr->attr.max_wr, attr->attr.max_sge, req.rq_spec, udata); if (rc) return rc;
srq->srq_limit = attr->attr.srq_limit;
rc = ionic_create_srq_cmd(dev, ctx, srq, pd, &buf, udma_mask); if (rc) goto err_cmd;
ionic_queue_dbell_init(&srq->rq.q, srq->rq.qid);
if (udata) { resp.srqid = srq->rq.qid; resp.udma_idx = srq->udma_idx;
if (srq->rq.cmb & IONIC_CMB_ENABLE) { ionic_rq_mmap_cmb(dev, ctx, &srq->rq, &resp.rq_cmb_offset); if (!srq->rq.mmap_cmb) { rc = -ENOMEM; goto err_srq_cmd; }
resp.rq_cmb = srq->rq.cmb; }
rc = ib_respond_udata(udata, resp); if (rc) goto err_srq_cmd; }
kref_init(&srq->kref); init_completion(&srq->rel_comp);
rc = xa_insert_irq(&dev->srq_tbl, srq->rq.qid, srq, GFP_KERNEL); if (rc) goto err_srq_cmd;
ionic_pgtbl_unbuf(dev, &buf);
attr->attr.max_wr = srq->rq.q.mask;
return 0;
err_srq_cmd: ionic_destroy_srq_cmd(dev, srq->rq.qid);err_cmd: ionic_pgtbl_unbuf(dev, &buf); ionic_rq_destroy(dev, ctx, &srq->rq);
return rc;}
int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata){ struct ionic_ctx *ctx = rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); struct ionic_srq *srq = to_ionic_srq(ibsrq); int rc;
if (xa_erase_irq(&dev->srq_tbl, srq->rq.qid)) { kref_put(&srq->kref, ionic_srq_complete); wait_for_completion(&srq->rel_comp); }
rc = ionic_destroy_srq_cmd(dev, srq->rq.qid); if (rc) return rc;
ionic_rq_destroy(dev, ctx, &srq->rq);
return 0;}
int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr){ struct ionic_srq *srq = to_ionic_srq(ibsrq);
srq_attr->max_wr = srq->rq.q.mask; srq_attr->max_sge = ionic_v1_recv_wqe_max_sge(srq->rq.q.stride_log2, srq->rq.spec, srq->rq.cmb & IONIC_CMB_EXPDB); srq_attr->srq_limit = srq->srq_limit;
return 0;}
int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); struct ionic_srq *srq = to_ionic_srq(ibsrq); int rc;
if (attr_mask & IB_SRQ_MAX_WR) return -EINVAL;
if (attr_mask & IB_SRQ_LIMIT) { if (attr->srq_limit >= IONIC_MAX_SRQ_LIMIT) return -EINVAL;
rc = ionic_modify_srq_cmd(dev, srq->rq.qid, attr->srq_limit); if (rc) return rc;
srq->srq_limit = attr->srq_limit; }
return 0;}