Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
#include <linux/module.h>#include <linux/printk.h>#include <rdma/ib_addr.h>#include <rdma/ib_user_verbs.h>
#include "ionic_fw.h"#include "ionic_ibdev.h"
#define IONIC_OP(version, opname) \ ((version) < 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname)
static bool ionic_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_v1_cqe **cqe){ struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q);
if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) return false;
/* Prevent out-of-order reads of the CQE */ dma_rmb();
*cqe = qcqe;
return true;}
static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc){ struct ionic_rq_meta *meta; struct ionic_v1_wqe *wqe; u64 wqe_idx;
if (!qp->rq.flush) return 0;
if (ionic_queue_empty(&qp->rq.q)) return 0;
wqe = ionic_queue_at_cons(&qp->rq.q); wqe_idx = le64_to_cpu(wqe->base.wqe_idx);
/* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu invalid\n", qp->qpid, (unsigned long long)wqe_idx); return -EIO; }
/* wqe_idx must indicate a request that is outstanding */ meta = &qp->rq.meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu not posted\n", qp->qpid, (unsigned long long)wqe_idx); return -EIO; }
ionic_queue_consume(&qp->rq.q);
memset(wc, 0, sizeof(*wc));
wc->status = IB_WC_WR_FLUSH_ERR; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp;
meta->next = qp->rq.meta_head; qp->rq.meta_head = meta;
return 1;}
static int ionic_flush_recv_many(struct ionic_qp *qp, struct ib_wc *wc, int nwc){ int rc = 0, npolled = 0;
while (npolled < nwc) { rc = ionic_flush_recv(qp, wc + npolled); if (rc <= 0) break;
npolled += rc; }
return npolled ?: rc;}
static int ionic_flush_send(struct ionic_qp *qp, struct ib_wc *wc){ struct ionic_sq_meta *meta;
if (!qp->sq_flush) return 0;
if (ionic_queue_empty(&qp->sq)) return 0;
meta = &qp->sq_meta[qp->sq.cons];
ionic_queue_consume(&qp->sq);
memset(wc, 0, sizeof(*wc));
wc->status = IB_WC_WR_FLUSH_ERR; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp;
return 1;}
static int ionic_flush_send_many(struct ionic_qp *qp, struct ib_wc *wc, int nwc){ int rc = 0, npolled = 0;
while (npolled < nwc) { rc = ionic_flush_send(qp, wc + npolled); if (rc <= 0) break;
npolled += rc; }
return npolled ?: rc;}
static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *cqe_qp, struct ionic_v1_cqe *cqe, struct ib_wc *wc){ struct ionic_qp *qp = NULL; struct ionic_rq_meta *meta; u16 vlan_tag, wqe_idx; u32 src_qpn, st_len; u8 op;
if (cqe_qp->rq.flush) return 0;
qp = cqe_qp;
st_len = be32_to_cpu(cqe->status_length);
/* ignore wqe_idx in case of flush error */ if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { cqe_qp->rq.flush = true; cq->flush = true; list_move_tail(&qp->rq.cq_flush, &cq->flush_rq);
/* posted recvs (if any) flushed by ionic_flush_recv */ return 0; }
/* there had better be something in the recv queue to complete */ if (ionic_queue_empty(&qp->rq.q)) { ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid); return -EIO; }
wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK; /* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u invalid\n", qp->qpid, wqe_idx); return -EIO; }
/* wqe_idx must indicate a request that is outstanding */ meta = &qp->rq.meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u not posted\n", qp->qpid, wqe_idx); return -EIO; }
meta->next = qp->rq.meta_head; qp->rq.meta_head = meta;
memset(wc, 0, sizeof(*wc));
wc->wr_id = meta->wrid;
wc->qp = &cqe_qp->ibqp;
if (ionic_v1_cqe_error(cqe)) { wc->vendor_err = st_len; wc->status = ionic_to_ib_status(st_len);
cqe_qp->rq.flush = true; cq->flush = true; list_move_tail(&qp->rq.cq_flush, &cq->flush_rq);
ibdev_warn(&dev->ibdev, "qp %d recv cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, BIT(cq->q.stride_log2), true); goto out; }
wc->vendor_err = 0; wc->status = IB_WC_SUCCESS;
src_qpn = be32_to_cpu(cqe->recv.src_qpn_op); op = src_qpn >> IONIC_V1_CQE_RECV_OP_SHIFT;
src_qpn &= IONIC_V1_CQE_RECV_QPN_MASK; op &= IONIC_V1_CQE_RECV_OP_MASK;
wc->opcode = IB_WC_RECV; switch (op) { case IONIC_V1_CQE_RECV_OP_RDMA_IMM: wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; wc->wc_flags |= IB_WC_WITH_IMM; wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ break; case IONIC_V1_CQE_RECV_OP_SEND_IMM: wc->wc_flags |= IB_WC_WITH_IMM; wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ break; case IONIC_V1_CQE_RECV_OP_SEND_INV: wc->wc_flags |= IB_WC_WITH_INVALIDATE; wc->ex.invalidate_rkey = be32_to_cpu(cqe->recv.imm_data_rkey); break; }
wc->byte_len = st_len; wc->src_qp = src_qpn;
if (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_GSI) { wc->wc_flags |= IB_WC_GRH | IB_WC_WITH_SMAC; ether_addr_copy(wc->smac, cqe->recv.src_mac);
wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; if (ionic_v1_cqe_recv_is_ipv4(cqe)) wc->network_hdr_type = RDMA_NETWORK_IPV4; else wc->network_hdr_type = RDMA_NETWORK_IPV6;
if (ionic_v1_cqe_recv_is_vlan(cqe)) wc->wc_flags |= IB_WC_WITH_VLAN;
/* vlan_tag in cqe will be valid from dpath even if no vlan */ vlan_tag = be16_to_cpu(cqe->recv.vlan_tag); wc->vlan_id = vlan_tag & 0xfff; /* 802.1q VID */ wc->sl = vlan_tag >> VLAN_PRIO_SHIFT; /* 802.1q PCP */ }
wc->pkey_index = 0; wc->port_num = 1;
out: ionic_queue_consume(&qp->rq.q);
return 1;}
static bool ionic_peek_send(struct ionic_qp *qp){ struct ionic_sq_meta *meta;
if (qp->sq_flush) return false;
/* completed all send queue requests */ if (ionic_queue_empty(&qp->sq)) return false;
meta = &qp->sq_meta[qp->sq.cons];
/* waiting for remote completion */ if (meta->remote && meta->seq == qp->sq_msn_cons) return false;
/* waiting for local completion */ if (!meta->remote && !meta->local_comp) return false;
return true;}
static int ionic_poll_send(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *qp, struct ib_wc *wc){ struct ionic_sq_meta *meta;
if (qp->sq_flush) return 0;
do { /* completed all send queue requests */ if (ionic_queue_empty(&qp->sq)) goto out_empty;
meta = &qp->sq_meta[qp->sq.cons];
/* waiting for remote completion */ if (meta->remote && meta->seq == qp->sq_msn_cons) goto out_empty;
/* waiting for local completion */ if (!meta->remote && !meta->local_comp) goto out_empty;
ionic_queue_consume(&qp->sq);
/* produce wc only if signaled or error status */ } while (!meta->signal && meta->ibsts == IB_WC_SUCCESS);
memset(wc, 0, sizeof(*wc));
wc->status = meta->ibsts; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp;
if (meta->ibsts == IB_WC_SUCCESS) { wc->byte_len = meta->len; wc->opcode = meta->ibop; } else { wc->vendor_err = meta->len;
qp->sq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); }
return 1;
out_empty: if (qp->sq_flush_rcvd) { qp->sq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); } return 0;}
static int ionic_poll_send_many(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *qp, struct ib_wc *wc, int nwc){ int rc = 0, npolled = 0;
while (npolled < nwc) { rc = ionic_poll_send(dev, cq, qp, wc + npolled); if (rc <= 0) break;
npolled += rc; }
return npolled ?: rc;}
static int ionic_validate_cons(u16 prod, u16 cons, u16 comp, u16 mask){ if (((prod - cons) & mask) <= ((comp - cons) & mask)) return -EIO;
return 0;}
static int ionic_comp_msn(struct ionic_qp *qp, struct ionic_v1_cqe *cqe){ struct ionic_sq_meta *meta; u16 cqe_seq, cqe_idx; int rc;
if (qp->sq_flush) return 0;
cqe_seq = be32_to_cpu(cqe->send.msg_msn) & qp->sq.mask;
rc = ionic_validate_cons(qp->sq_msn_prod, qp->sq_msn_cons, cqe_seq - 1, qp->sq.mask); if (rc) { ibdev_warn(qp->ibqp.device, "qp %u bad msn %#x seq %u for prod %u cons %u\n", qp->qpid, be32_to_cpu(cqe->send.msg_msn), cqe_seq, qp->sq_msn_prod, qp->sq_msn_cons); return rc; }
qp->sq_msn_cons = cqe_seq;
if (ionic_v1_cqe_error(cqe)) { cqe_idx = qp->sq_msn_idx[(cqe_seq - 1) & qp->sq.mask];
meta = &qp->sq_meta[cqe_idx]; meta->len = be32_to_cpu(cqe->status_length); meta->ibsts = ionic_to_ib_status(meta->len);
ibdev_warn(qp->ibqp.device, "qp %d msn cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, sizeof(*cqe), true); }
return 0;}
static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe){ struct ionic_sq_meta *meta; u16 wqe_idx; u32 st_len;
if (qp->sq_flush) return 0;
st_len = be32_to_cpu(cqe->status_length);
if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { /* * Flush cqe does not consume a wqe on the device, and maybe * no such work request is posted. * * The driver should begin flushing after the last indicated * normal or error completion. Here, only set a hint that the * flush request was indicated. In poll_send, if nothing more * can be polled normally, then begin flushing. */ qp->sq_flush_rcvd = true; return 0; }
wqe_idx = le64_to_cpu(cqe->send.npg_wqe_idx_timestamp) & qp->sq.mask; meta = &qp->sq_meta[wqe_idx]; meta->local_comp = true;
if (ionic_v1_cqe_error(cqe)) { meta->len = st_len; meta->ibsts = ionic_to_ib_status(st_len); meta->remote = false; ibdev_warn(qp->ibqp.device, "qp %d npg cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, sizeof(*cqe), true); }
return 0;}
static void ionic_reserve_sync_cq(struct ionic_ibdev *dev, struct ionic_cq *cq){ if (!ionic_queue_empty(&cq->q)) { cq->credit += ionic_queue_length(&cq->q); cq->q.cons = cq->q.prod;
ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, ionic_queue_dbell_val(&cq->q)); }}
static void ionic_reserve_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, int spend){ cq->credit -= spend;
if (cq->credit <= 0) ionic_reserve_sync_cq(dev, cq);}
static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, int nwc, struct ib_wc *wc){ struct ionic_qp *qp, *qp_next; struct ionic_v1_cqe *cqe; int rc = 0, npolled = 0; unsigned long irqflags; u32 qtf, qid; bool peek; u8 type;
if (nwc < 1) return 0;
spin_lock_irqsave(&cq->lock, irqflags);
/* poll already indicated work completions for send queue */ list_for_each_entry_safe(qp, qp_next, &cq->poll_sq, cq_poll_sq) { if (npolled == nwc) goto out;
spin_lock(&qp->sq_lock); rc = ionic_poll_send_many(dev, cq, qp, wc + npolled, nwc - npolled); spin_unlock(&qp->sq_lock);
if (rc > 0) npolled += rc;
if (npolled < nwc) list_del_init(&qp->cq_poll_sq); }
/* poll for more work completions */ while (likely(ionic_next_cqe(dev, cq, &cqe))) { if (npolled == nwc) goto out;
qtf = ionic_v1_cqe_qtf(cqe); qid = ionic_v1_cqe_qtf_qid(qtf); type = ionic_v1_cqe_qtf_type(qtf);
/* * Safe to access QP without additional reference here as, * 1. We hold cq->lock throughout * 2. ionic_destroy_qp() acquires the same cq->lock before cleanup * 3. QP is removed from qp_tbl before any cleanup begins * This ensures no concurrent access between polling and destruction. */ qp = xa_load(&dev->qp_tbl, qid); if (unlikely(!qp)) { ibdev_dbg(&dev->ibdev, "missing qp for qid %u\n", qid); goto cq_next; }
switch (type) { case IONIC_V1_CQE_TYPE_RECV: if (qp->srq) { ibdev_dbg(&dev->ibdev, "srq recv cqe for qp %u\n", qid); goto cq_next; }
spin_lock(&qp->rq.lock); rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled); spin_unlock(&qp->rq.lock);
if (rc < 0) goto out;
npolled += rc;
break;
case IONIC_V1_CQE_TYPE_SEND_MSN: spin_lock(&qp->sq_lock); rc = ionic_comp_msn(qp, cqe); if (!rc) { rc = ionic_poll_send_many(dev, cq, qp, wc + npolled, nwc - npolled); peek = ionic_peek_send(qp); } spin_unlock(&qp->sq_lock);
if (rc < 0) goto out;
npolled += rc;
if (peek) list_move_tail(&qp->cq_poll_sq, &cq->poll_sq); break;
case IONIC_V1_CQE_TYPE_SEND_NPG: spin_lock(&qp->sq_lock); rc = ionic_comp_npg(qp, cqe); if (!rc) { rc = ionic_poll_send_many(dev, cq, qp, wc + npolled, nwc - npolled); peek = ionic_peek_send(qp); } spin_unlock(&qp->sq_lock);
if (rc < 0) goto out;
npolled += rc;
if (peek) list_move_tail(&qp->cq_poll_sq, &cq->poll_sq); break;
default: ibdev_warn(&dev->ibdev, "unexpected cqe type %u\n", type); rc = -EIO; goto out; }
cq_next: ionic_queue_produce(&cq->q); cq->color = ionic_color_wrap(cq->q.prod, cq->color); }
/* lastly, flush send and recv queues */ if (likely(!cq->flush)) goto out;
cq->flush = false;
list_for_each_entry_safe(qp, qp_next, &cq->flush_sq, cq_flush_sq) { if (npolled == nwc) goto out;
spin_lock(&qp->sq_lock); rc = ionic_flush_send_many(qp, wc + npolled, nwc - npolled); spin_unlock(&qp->sq_lock);
if (rc > 0) npolled += rc;
if (npolled < nwc) list_del_init(&qp->cq_flush_sq); else cq->flush = true; }
list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, rq.cq_flush) { if (npolled == nwc) goto out;
spin_lock(&qp->rq.lock); rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled); spin_unlock(&qp->rq.lock);
if (rc > 0) npolled += rc;
if (npolled < nwc) list_del_init(&qp->rq.cq_flush); else cq->flush = true; }
out: /* in case credit was depleted (more work posted than cq depth) */ if (cq->credit <= 0) ionic_reserve_sync_cq(dev, cq);
spin_unlock_irqrestore(&cq->lock, irqflags);
return npolled ?: rc;}
int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc){ struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); int rc_tmp, rc = 0, npolled = 0; int cq_i, cq_x, cq_ix;
cq_x = vcq->poll_idx; vcq->poll_idx ^= dev->lif_cfg.udma_count - 1;
for (cq_i = 0; npolled < nwc && cq_i < dev->lif_cfg.udma_count; ++cq_i) { cq_ix = cq_i ^ cq_x;
if (!(vcq->udma_mask & BIT(cq_ix))) continue;
rc_tmp = ionic_poll_vcq_cq(dev, &vcq->cq[cq_ix], nwc - npolled, wc + npolled);
if (rc_tmp >= 0) npolled += rc_tmp; else if (!rc) rc = rc_tmp; }
return npolled ?: rc;}
static int ionic_req_notify_vcq_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, enum ib_cq_notify_flags flags){ u64 dbell_val = cq->q.dbell;
if (flags & IB_CQ_SOLICITED) { cq->arm_sol_prod = ionic_queue_next(&cq->q, cq->arm_sol_prod); dbell_val |= cq->arm_sol_prod | IONIC_CQ_RING_SOL; } else { cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); dbell_val |= cq->arm_any_prod | IONIC_CQ_RING_ARM; }
ionic_reserve_sync_cq(dev, cq);
ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, dbell_val);
/* * IB_CQ_REPORT_MISSED_EVENTS: * * The queue index in ring zero guarantees no missed events. * * Here, we check if the color bit in the next cqe is flipped. If it * is flipped, then progress can be made by immediately polling the cq. * Still, the cq will be armed, and an event will be generated. The cq * may be empty when polled after the event, because the next poll * after arming the cq can empty it. */ return (flags & IB_CQ_REPORT_MISSED_EVENTS) && cq->color == ionic_v1_cqe_color(ionic_queue_at_prod(&cq->q));}
int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags){ struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); int rc = 0, cq_i;
for (cq_i = 0; cq_i < dev->lif_cfg.udma_count; ++cq_i) { if (!(vcq->udma_mask & BIT(cq_i))) continue;
if (ionic_req_notify_vcq_cq(dev, &vcq->cq[cq_i], flags)) rc = 1; }
return rc;}
static s64 ionic_prep_inline(void *data, u32 max_data, const struct ib_sge *ib_sgl, int num_sge){ static const s64 bit_31 = 1u << 31; s64 len = 0, sg_len; int sg_i;
for (sg_i = 0; sg_i < num_sge; ++sg_i) { sg_len = ib_sgl[sg_i].length;
/* sge length zero means 2GB */ if (unlikely(sg_len == 0)) sg_len = bit_31;
/* greater than max inline data is invalid */ if (unlikely(len + sg_len > max_data)) return -EINVAL;
memcpy(data + len, (void *)ib_sgl[sg_i].addr, sg_len);
len += sg_len; }
return len;}
static s64 ionic_prep_pld(struct ionic_v1_wqe *wqe, union ionic_v1_pld *pld, int spec, u32 max_sge, const struct ib_sge *ib_sgl, int num_sge){ static const s64 bit_31 = 1l << 31; struct ionic_sge *sgl; __be32 *spec32 = NULL; __be16 *spec16 = NULL; s64 len = 0, sg_len; int sg_i = 0;
if (unlikely(num_sge < 0 || (u32)num_sge > max_sge)) return -EINVAL;
if (spec && num_sge > IONIC_V1_SPEC_FIRST_SGE) { sg_i = IONIC_V1_SPEC_FIRST_SGE;
if (num_sge > 8) { wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SPEC16); spec16 = pld->spec16; } else { wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SPEC32); spec32 = pld->spec32; } }
sgl = &pld->sgl[sg_i];
for (sg_i = 0; sg_i < num_sge; ++sg_i) { sg_len = ib_sgl[sg_i].length;
/* sge length zero means 2GB */ if (unlikely(sg_len == 0)) sg_len = bit_31;
/* greater than 2GB data is invalid */ if (unlikely(len + sg_len > bit_31)) return -EINVAL;
sgl[sg_i].va = cpu_to_be64(ib_sgl[sg_i].addr); sgl[sg_i].len = cpu_to_be32(sg_len); sgl[sg_i].lkey = cpu_to_be32(ib_sgl[sg_i].lkey);
if (spec32) { spec32[sg_i] = sgl[sg_i].len; } else if (spec16) { if (unlikely(sg_len > U16_MAX)) return -EINVAL; spec16[sg_i] = cpu_to_be16(sg_len); }
len += sg_len; }
return len;}
static void ionic_prep_base(struct ionic_qp *qp, const struct ib_send_wr *wr, struct ionic_sq_meta *meta, struct ionic_v1_wqe *wqe){ meta->wrid = wr->wr_id; meta->ibsts = IB_WC_SUCCESS; meta->signal = false; meta->local_comp = false;
wqe->base.wqe_idx = cpu_to_le64(qp->sq.prod);
if (wr->send_flags & IB_SEND_FENCE) wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_FENCE);
if (wr->send_flags & IB_SEND_SOLICITED) wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SOL);
if (qp->sig_all || wr->send_flags & IB_SEND_SIGNALED) { wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SIG); meta->signal = true; }
meta->seq = qp->sq_msn_prod; meta->remote = qp->ibqp.qp_type != IB_QPT_UD && qp->ibqp.qp_type != IB_QPT_GSI && !ionic_ibop_is_local(wr->opcode);
if (meta->remote) { qp->sq_msn_idx[meta->seq] = qp->sq.prod; qp->sq_msn_prod = ionic_queue_next(&qp->sq, qp->sq_msn_prod); }
ionic_queue_produce(&qp->sq);}
static int ionic_prep_common(struct ionic_qp *qp, const struct ib_send_wr *wr, struct ionic_sq_meta *meta, struct ionic_v1_wqe *wqe){ s64 signed_len; u32 mval;
if (wr->send_flags & IB_SEND_INLINE) { wqe->base.num_sge_key = 0; wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_INL); mval = ionic_v1_send_wqe_max_data(qp->sq.stride_log2, false); signed_len = ionic_prep_inline(wqe->common.pld.data, mval, wr->sg_list, wr->num_sge); } else { wqe->base.num_sge_key = wr->num_sge; mval = ionic_v1_send_wqe_max_sge(qp->sq.stride_log2, qp->sq_spec, false); signed_len = ionic_prep_pld(wqe, &wqe->common.pld, qp->sq_spec, mval, wr->sg_list, wr->num_sge); }
if (unlikely(signed_len < 0)) return signed_len;
meta->len = signed_len; wqe->common.length = cpu_to_be32(signed_len);
ionic_prep_base(qp, wr, meta, wqe);
return 0;}
static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe){ memset(wqe, 0, 1u << qp->sq.stride_log2);}
static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe){ memset(wqe, 0, 1u << qp->rq.q.stride_log2);}
static int ionic_prep_send(struct ionic_qp *qp, const struct ib_send_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); struct ionic_sq_meta *meta; struct ionic_v1_wqe *wqe;
meta = &qp->sq_meta[qp->sq.prod]; wqe = ionic_queue_at_prod(&qp->sq);
ionic_prep_sq_wqe(qp, wqe);
meta->ibop = IB_WC_SEND;
switch (wr->opcode) { case IB_WR_SEND: wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND); break; case IB_WR_SEND_WITH_IMM: wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND_IMM); wqe->base.imm_data_key = wr->ex.imm_data; break; case IB_WR_SEND_WITH_INV: wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND_INV); wqe->base.imm_data_key = cpu_to_be32(wr->ex.invalidate_rkey); break; default: return -EINVAL; }
return ionic_prep_common(qp, wr, meta, wqe);}
static int ionic_prep_send_ud(struct ionic_qp *qp, const struct ib_ud_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); struct ionic_sq_meta *meta; struct ionic_v1_wqe *wqe; struct ionic_ah *ah;
if (unlikely(!wr->ah)) return -EINVAL;
ah = to_ionic_ah(wr->ah);
meta = &qp->sq_meta[qp->sq.prod]; wqe = ionic_queue_at_prod(&qp->sq);
ionic_prep_sq_wqe(qp, wqe);
wqe->common.send.ah_id = cpu_to_be32(ah->ahid); wqe->common.send.dest_qpn = cpu_to_be32(wr->remote_qpn); wqe->common.send.dest_qkey = cpu_to_be32(wr->remote_qkey);
meta->ibop = IB_WC_SEND;
switch (wr->wr.opcode) { case IB_WR_SEND: wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND); break; case IB_WR_SEND_WITH_IMM: wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND_IMM); wqe->base.imm_data_key = wr->wr.ex.imm_data; break; default: return -EINVAL; }
return ionic_prep_common(qp, &wr->wr, meta, wqe);}
static int ionic_prep_rdma(struct ionic_qp *qp, const struct ib_rdma_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); struct ionic_sq_meta *meta; struct ionic_v1_wqe *wqe;
meta = &qp->sq_meta[qp->sq.prod]; wqe = ionic_queue_at_prod(&qp->sq);
ionic_prep_sq_wqe(qp, wqe);
meta->ibop = IB_WC_RDMA_WRITE;
switch (wr->wr.opcode) { case IB_WR_RDMA_READ: if (wr->wr.send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE)) return -EINVAL; meta->ibop = IB_WC_RDMA_READ; wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, RDMA_READ); break; case IB_WR_RDMA_WRITE: if (wr->wr.send_flags & IB_SEND_SOLICITED) return -EINVAL; wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, RDMA_WRITE); break; case IB_WR_RDMA_WRITE_WITH_IMM: wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, RDMA_WRITE_IMM); wqe->base.imm_data_key = wr->wr.ex.imm_data; break; default: return -EINVAL; }
wqe->common.rdma.remote_va_high = cpu_to_be32(wr->remote_addr >> 32); wqe->common.rdma.remote_va_low = cpu_to_be32(wr->remote_addr); wqe->common.rdma.remote_rkey = cpu_to_be32(wr->rkey);
return ionic_prep_common(qp, &wr->wr, meta, wqe);}
static int ionic_prep_atomic(struct ionic_qp *qp, const struct ib_atomic_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); struct ionic_sq_meta *meta; struct ionic_v1_wqe *wqe;
if (wr->wr.num_sge != 1 || wr->wr.sg_list[0].length != 8) return -EINVAL;
if (wr->wr.send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE)) return -EINVAL;
meta = &qp->sq_meta[qp->sq.prod]; wqe = ionic_queue_at_prod(&qp->sq);
ionic_prep_sq_wqe(qp, wqe);
meta->ibop = IB_WC_RDMA_WRITE;
switch (wr->wr.opcode) { case IB_WR_ATOMIC_CMP_AND_SWP: meta->ibop = IB_WC_COMP_SWAP; wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, ATOMIC_CS); wqe->atomic.swap_add_high = cpu_to_be32(wr->swap >> 32); wqe->atomic.swap_add_low = cpu_to_be32(wr->swap); wqe->atomic.compare_high = cpu_to_be32(wr->compare_add >> 32); wqe->atomic.compare_low = cpu_to_be32(wr->compare_add); break; case IB_WR_ATOMIC_FETCH_AND_ADD: meta->ibop = IB_WC_FETCH_ADD; wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, ATOMIC_FA); wqe->atomic.swap_add_high = cpu_to_be32(wr->compare_add >> 32); wqe->atomic.swap_add_low = cpu_to_be32(wr->compare_add); break; default: return -EINVAL; }
wqe->atomic.remote_va_high = cpu_to_be32(wr->remote_addr >> 32); wqe->atomic.remote_va_low = cpu_to_be32(wr->remote_addr); wqe->atomic.remote_rkey = cpu_to_be32(wr->rkey);
wqe->base.num_sge_key = 1; wqe->atomic.sge.va = cpu_to_be64(wr->wr.sg_list[0].addr); wqe->atomic.sge.len = cpu_to_be32(8); wqe->atomic.sge.lkey = cpu_to_be32(wr->wr.sg_list[0].lkey);
return ionic_prep_common(qp, &wr->wr, meta, wqe);}
static int ionic_prep_inv(struct ionic_qp *qp, const struct ib_send_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); struct ionic_sq_meta *meta; struct ionic_v1_wqe *wqe;
if (wr->send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE)) return -EINVAL;
meta = &qp->sq_meta[qp->sq.prod]; wqe = ionic_queue_at_prod(&qp->sq);
ionic_prep_sq_wqe(qp, wqe);
wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, LOCAL_INV); wqe->base.imm_data_key = cpu_to_be32(wr->ex.invalidate_rkey);
meta->len = 0; meta->ibop = IB_WC_LOCAL_INV;
ionic_prep_base(qp, wr, meta, wqe);
return 0;}
static int ionic_prep_reg(struct ionic_qp *qp, const struct ib_reg_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); struct ionic_mr *mr = to_ionic_mr(wr->mr); struct ionic_sq_meta *meta; struct ionic_v1_wqe *wqe; __le64 dma_addr; int flags;
if (wr->wr.send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE)) return -EINVAL;
/* must call ib_map_mr_sg before posting reg wr */ if (!mr->buf.tbl_pages) return -EINVAL;
meta = &qp->sq_meta[qp->sq.prod]; wqe = ionic_queue_at_prod(&qp->sq);
ionic_prep_sq_wqe(qp, wqe);
flags = to_ionic_mr_flags(wr->access);
wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, REG_MR); wqe->base.num_sge_key = wr->key; wqe->base.imm_data_key = cpu_to_be32(mr->ibmr.lkey); wqe->reg_mr.va = cpu_to_be64(mr->ibmr.iova); wqe->reg_mr.length = cpu_to_be64(mr->ibmr.length); wqe->reg_mr.offset = ionic_pgtbl_off(&mr->buf, mr->ibmr.iova); dma_addr = ionic_pgtbl_dma(&mr->buf, mr->ibmr.iova); wqe->reg_mr.dma_addr = cpu_to_be64(le64_to_cpu(dma_addr));
wqe->reg_mr.map_count = cpu_to_be32(mr->buf.tbl_pages); wqe->reg_mr.flags = cpu_to_be16(flags); wqe->reg_mr.dir_size_log2 = 0; wqe->reg_mr.page_size_log2 = order_base_2(mr->ibmr.page_size);
meta->len = 0; meta->ibop = IB_WC_REG_MR;
ionic_prep_base(qp, &wr->wr, meta, wqe);
return 0;}
static int ionic_prep_one_rc(struct ionic_qp *qp, const struct ib_send_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); int rc = 0;
switch (wr->opcode) { case IB_WR_SEND: case IB_WR_SEND_WITH_IMM: case IB_WR_SEND_WITH_INV: rc = ionic_prep_send(qp, wr); break; case IB_WR_RDMA_READ: case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: rc = ionic_prep_rdma(qp, rdma_wr(wr)); break; case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: rc = ionic_prep_atomic(qp, atomic_wr(wr)); break; case IB_WR_LOCAL_INV: rc = ionic_prep_inv(qp, wr); break; case IB_WR_REG_MR: rc = ionic_prep_reg(qp, reg_wr(wr)); break; default: ibdev_dbg(&dev->ibdev, "invalid opcode %d\n", wr->opcode); rc = -EINVAL; }
return rc;}
static int ionic_prep_one_ud(struct ionic_qp *qp, const struct ib_send_wr *wr){ struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device); int rc = 0;
switch (wr->opcode) { case IB_WR_SEND: case IB_WR_SEND_WITH_IMM: rc = ionic_prep_send_ud(qp, ud_wr(wr)); break; default: ibdev_dbg(&dev->ibdev, "invalid opcode %d\n", wr->opcode); rc = -EINVAL; }
return rc;}
static int ionic_prep_recv(struct ionic_qp *qp, const struct ib_recv_wr *wr){ struct ionic_rq_meta *meta; struct ionic_v1_wqe *wqe; s64 signed_len; u32 mval;
wqe = ionic_queue_at_prod(&qp->rq.q);
/* if wqe is owned by device, caller can try posting again soon */ if (wqe->base.flags & cpu_to_be16(IONIC_V1_FLAG_FENCE)) return -EAGAIN;
meta = qp->rq.meta_head; if (unlikely(meta == IONIC_META_LAST) || unlikely(meta == IONIC_META_POSTED)) return -EIO;
ionic_prep_rq_wqe(qp, wqe);
mval = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec, false); signed_len = ionic_prep_pld(wqe, &wqe->recv.pld, qp->rq.spec, mval, wr->sg_list, wr->num_sge); if (signed_len < 0) return signed_len;
meta->wrid = wr->wr_id;
wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq.meta); wqe->base.num_sge_key = wr->num_sge;
/* total length for recv goes in base imm_data_key */ wqe->base.imm_data_key = cpu_to_be32(signed_len);
ionic_queue_produce(&qp->rq.q);
qp->rq.meta_head = meta->next; meta->next = IONIC_META_POSTED;
return 0;}
static int ionic_post_send_common(struct ionic_ibdev *dev, struct ionic_vcq *vcq, struct ionic_cq *cq, struct ionic_qp *qp, const struct ib_send_wr *wr, const struct ib_send_wr **bad){ unsigned long irqflags; bool notify = false; int spend, rc = 0;
if (!bad) return -EINVAL;
if (!qp->has_sq) { *bad = wr; return -EINVAL; }
if (qp->state < IB_QPS_RTS) { *bad = wr; return -EINVAL; }
spin_lock_irqsave(&qp->sq_lock, irqflags);
while (wr) { if (ionic_queue_full(&qp->sq)) { ibdev_dbg(&dev->ibdev, "queue full"); rc = -ENOMEM; goto out; }
if (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_GSI) rc = ionic_prep_one_ud(qp, wr); else rc = ionic_prep_one_rc(qp, wr); if (rc) goto out;
wr = wr->next; }
out: spin_unlock_irqrestore(&qp->sq_lock, irqflags);
spin_lock_irqsave(&cq->lock, irqflags); spin_lock(&qp->sq_lock);
if (likely(qp->sq.prod != qp->sq_old_prod)) { /* ring cq doorbell just in time */ spend = (qp->sq.prod - qp->sq_old_prod) & qp->sq.mask; ionic_reserve_cq(dev, cq, spend);
qp->sq_old_prod = qp->sq.prod;
ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.sq_qtype, ionic_queue_dbell_val(&qp->sq)); }
if (qp->sq_flush) { notify = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); }
spin_unlock(&qp->sq_lock); spin_unlock_irqrestore(&cq->lock, irqflags);
if (notify && vcq->ibcq.comp_handler) vcq->ibcq.comp_handler(&vcq->ibcq, vcq->ibcq.cq_context);
*bad = wr; return rc;}
static int ionic_post_recv_common(struct ionic_ibdev *dev, struct ionic_vcq *vcq, struct ionic_cq *cq, struct ionic_qp *qp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad){ unsigned long irqflags; bool notify = false; int spend, rc = 0;
if (!bad) return -EINVAL;
if (qp->srq) { *bad = wr; return -EINVAL; }
if (qp->state < IB_QPS_INIT) { *bad = wr; return -EINVAL; }
spin_lock_irqsave(&qp->rq.lock, irqflags);
while (wr) { if (ionic_queue_full(&qp->rq.q)) { ibdev_dbg(&dev->ibdev, "queue full"); rc = -ENOMEM; goto out; }
rc = ionic_prep_recv(qp, wr); if (rc) goto out;
wr = wr->next; }
out: if (!cq) { spin_unlock_irqrestore(&qp->rq.lock, irqflags); goto out_unlocked; } spin_unlock_irqrestore(&qp->rq.lock, irqflags);
spin_lock_irqsave(&cq->lock, irqflags); spin_lock(&qp->rq.lock);
if (likely(qp->rq.q.prod != qp->rq.old_prod)) { /* ring cq doorbell just in time */ spend = (qp->rq.q.prod - qp->rq.old_prod) & qp->rq.q.mask; ionic_reserve_cq(dev, cq, spend);
qp->rq.old_prod = qp->rq.q.prod;
ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.rq_qtype, ionic_queue_dbell_val(&qp->rq.q)); }
if (qp->rq.flush) { notify = true; cq->flush = true; list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); }
spin_unlock(&qp->rq.lock); spin_unlock_irqrestore(&cq->lock, irqflags);
if (notify && vcq->ibcq.comp_handler) vcq->ibcq.comp_handler(&vcq->ibcq, vcq->ibcq.cq_context);
out_unlocked: *bad = wr; return rc;}
int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad){ struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); struct ionic_vcq *vcq = to_ionic_vcq(ibqp->send_cq); struct ionic_qp *qp = to_ionic_qp(ibqp); struct ionic_cq *cq = to_ionic_vcq_cq(ibqp->send_cq, qp->udma_idx);
return ionic_post_send_common(dev, vcq, cq, qp, wr, bad);}
int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad){ struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device); struct ionic_vcq *vcq = to_ionic_vcq(ibqp->recv_cq); struct ionic_qp *qp = to_ionic_qp(ibqp); struct ionic_cq *cq = to_ionic_vcq_cq(ibqp->recv_cq, qp->udma_idx);
return ionic_post_recv_common(dev, vcq, cq, qp, wr, bad);}