Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
#include <linux/interrupt.h>#include <linux/module.h>#include <linux/printk.h>
#include "ionic_fw.h"#include "ionic_ibdev.h"
#define IONIC_EQ_COUNT_MIN 4#define IONIC_AQ_COUNT_MIN 1
/* not a valid queue position or negative error status */#define IONIC_ADMIN_POSTED 0x10000
/* cpu can be held with irq disabled for COUNT * MS (for create/destroy_ah) */#define IONIC_ADMIN_BUSY_RETRY_COUNT 2000#define IONIC_ADMIN_BUSY_RETRY_MS 1
/* admin queue will be considered failed if a command takes longer */#define IONIC_ADMIN_TIMEOUT (HZ * 2)#define IONIC_ADMIN_WARN (HZ / 8)
/* will poll for admin cq to tolerate and report from missed event */#define IONIC_ADMIN_DELAY (HZ / 8)
/* work queue for polling the event queue and admin cq */struct workqueue_struct *ionic_evt_workq;
static void ionic_admin_timedout(struct ionic_aq *aq){ struct ionic_ibdev *dev = aq->dev; unsigned long irqflags; u16 pos;
spin_lock_irqsave(&aq->lock, irqflags); if (ionic_queue_empty(&aq->q)) goto out;
/* Reset ALL adminq if any one times out */ if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) queue_work(ionic_evt_workq, &dev->reset_work);
ibdev_err(&dev->ibdev, "admin command timed out, aq %d after: %ums\n", aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp));
pos = (aq->q.prod - 1) & aq->q.mask; if (pos == aq->q.cons) goto out;
ibdev_warn(&dev->ibdev, "admin pos %u (last posted)\n", pos); print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at(&aq->q, pos), BIT(aq->q.stride_log2), true);
out: spin_unlock_irqrestore(&aq->lock, irqflags);}
static void ionic_admin_reset_dwork(struct ionic_ibdev *dev){ if (atomic_read(&dev->admin_state) == IONIC_ADMIN_KILLED) return;
queue_delayed_work(ionic_evt_workq, &dev->admin_dwork, IONIC_ADMIN_DELAY);}
static void ionic_admin_reset_wdog(struct ionic_aq *aq){ if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) return;
aq->stamp = jiffies; ionic_admin_reset_dwork(aq->dev);}
static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_v1_cqe **cqe){ struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q);
if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) return false;
/* Prevent out-of-order reads of the CQE */ dma_rmb(); *cqe = qcqe;
return true;}
static void ionic_admin_poll_locked(struct ionic_aq *aq){ struct ionic_cq *cq = &aq->vcq->cq[0]; struct ionic_admin_wr *wr, *wr_next; struct ionic_ibdev *dev = aq->dev; u32 wr_strides, avlbl_strides; struct ionic_v1_cqe *cqe; u32 qtf, qid; u16 old_prod; u8 type;
lockdep_assert_held(&aq->lock);
if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) { list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) { INIT_LIST_HEAD(&wr->aq_ent); aq->q_wr[wr->status].wr = NULL; wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } INIT_LIST_HEAD(&aq->wr_prod);
list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) { INIT_LIST_HEAD(&wr->aq_ent); wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } INIT_LIST_HEAD(&aq->wr_post);
return; }
old_prod = cq->q.prod;
while (ionic_admin_next_cqe(dev, cq, &cqe)) { qtf = ionic_v1_cqe_qtf(cqe); qid = ionic_v1_cqe_qtf_qid(qtf); type = ionic_v1_cqe_qtf_type(qtf);
if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe type %u\n", type); goto cq_next; }
if (unlikely(qid != aq->aqid)) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe qid %u\n", qid); goto cq_next; }
if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) { ibdev_warn_ratelimited(&dev->ibdev, "bad idx %u cons %u qid %u\n", be16_to_cpu(cqe->admin.cmd_idx), aq->q.cons, qid); goto cq_next; }
if (unlikely(ionic_queue_empty(&aq->q))) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe for empty adminq\n"); goto cq_next; }
wr = aq->q_wr[aq->q.cons].wr; if (wr) { aq->q_wr[aq->q.cons].wr = NULL; list_del_init(&wr->aq_ent);
wr->cqe = *cqe; wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); }
ionic_queue_consume_entries(&aq->q, aq->q_wr[aq->q.cons].wqe_strides);
cq_next: ionic_queue_produce(&cq->q); cq->color = ionic_color_wrap(cq->q.prod, cq->color); }
if (old_prod != cq->q.prod) { ionic_admin_reset_wdog(aq); cq->q.cons = cq->q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, ionic_queue_dbell_val(&cq->q)); queue_work(ionic_evt_workq, &aq->work); } else if (!aq->armed) { aq->armed = true; cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, cq->q.dbell | IONIC_CQ_RING_ARM | cq->arm_any_prod); queue_work(ionic_evt_workq, &aq->work); }
if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE) return;
old_prod = aq->q.prod;
if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post)) ionic_admin_reset_wdog(aq);
if (list_empty(&aq->wr_post)) return;
do { u8 *src; int i, src_len; size_t stride_len;
wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr, aq_ent); wr_strides = (le16_to_cpu(wr->wqe.len) + ADMIN_WQE_HDR_LEN + (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2; avlbl_strides = ionic_queue_length_remaining(&aq->q);
if (wr_strides > avlbl_strides) break;
list_move(&wr->aq_ent, &aq->wr_prod); wr->status = aq->q.prod; aq->q_wr[aq->q.prod].wr = wr; aq->q_wr[aq->q.prod].wqe_strides = wr_strides;
src_len = le16_to_cpu(wr->wqe.len); src = (uint8_t *)&wr->wqe.cmd;
/* First stride */ memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe, ADMIN_WQE_HDR_LEN); stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN; if (stride_len > src_len) stride_len = src_len; memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN, src, stride_len); ibdev_dbg(&dev->ibdev, "post admin prod %u (%u strides)\n", aq->q.prod, wr_strides); print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at_prod(&aq->q), BIT(aq->q.stride_log2), true); ionic_queue_produce(&aq->q);
/* Remaining strides */ for (i = stride_len; i < src_len; i += stride_len) { stride_len = ADMIN_WQE_STRIDE;
if (i + stride_len > src_len) stride_len = src_len - i;
memcpy(ionic_queue_at_prod(&aq->q), src + i, stride_len); print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at_prod(&aq->q), BIT(aq->q.stride_log2), true); ionic_queue_produce(&aq->q); } } while (!list_empty(&aq->wr_post));
if (old_prod != aq->q.prod) ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.aq_qtype, ionic_queue_dbell_val(&aq->q));}
static void ionic_admin_dwork(struct work_struct *ws){ struct ionic_ibdev *dev = container_of(ws, struct ionic_ibdev, admin_dwork.work); struct ionic_aq *aq, *bad_aq = NULL; bool do_reschedule = false; unsigned long irqflags; bool do_reset = false; u16 pos; int i;
for (i = 0; i < dev->lif_cfg.aq_count; i++) { aq = dev->aq_vec[i];
spin_lock_irqsave(&aq->lock, irqflags);
if (ionic_queue_empty(&aq->q)) goto next_aq;
/* Reschedule if any queue has outstanding work */ do_reschedule = true;
if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN)) /* Warning threshold not met, nothing to do */ goto next_aq;
/* See if polling now makes some progress */ pos = aq->q.cons; ionic_admin_poll_locked(aq); if (pos != aq->q.cons) { ibdev_dbg(&dev->ibdev, "missed event for acq %d\n", aq->cqid); goto next_aq; }
if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_TIMEOUT)) { /* Timeout threshold not met */ ibdev_dbg(&dev->ibdev, "no progress after %ums\n", (u32)jiffies_to_msecs(jiffies - aq->stamp)); goto next_aq; }
/* Queue timed out */ bad_aq = aq; do_reset = true;next_aq: spin_unlock_irqrestore(&aq->lock, irqflags); }
if (do_reset) /* Reset RDMA lif on a timeout */ ionic_admin_timedout(bad_aq); else if (do_reschedule) /* Try to poll again later */ ionic_admin_reset_dwork(dev);}
static void ionic_admin_work(struct work_struct *ws){ struct ionic_aq *aq = container_of(ws, struct ionic_aq, work); unsigned long irqflags;
spin_lock_irqsave(&aq->lock, irqflags); ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags);}
static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr){ unsigned long irqflags; bool poll;
wr->status = IONIC_ADMIN_POSTED; wr->aq = aq;
spin_lock_irqsave(&aq->lock, irqflags); poll = list_empty(&aq->wr_post); list_add(&wr->aq_ent, &aq->wr_post); if (poll) ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags);}
void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr){ int aq_idx;
/* Use cpu id for the adminq selection */ aq_idx = raw_smp_processor_id() % dev->lif_cfg.aq_count; ionic_admin_post_aq(dev->aq_vec[aq_idx], wr);}
static void ionic_admin_cancel(struct ionic_admin_wr *wr){ struct ionic_aq *aq = wr->aq; unsigned long irqflags;
spin_lock_irqsave(&aq->lock, irqflags);
if (!list_empty(&wr->aq_ent)) { list_del(&wr->aq_ent); if (wr->status != IONIC_ADMIN_POSTED) aq->q_wr[wr->status].wr = NULL; }
spin_unlock_irqrestore(&aq->lock, irqflags);}
static int ionic_admin_busy_wait(struct ionic_admin_wr *wr){ struct ionic_aq *aq = wr->aq; unsigned long irqflags; int try_i;
for (try_i = 0; try_i < IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) { if (completion_done(&wr->work)) return 0;
mdelay(IONIC_ADMIN_BUSY_RETRY_MS);
spin_lock_irqsave(&aq->lock, irqflags); ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags); }
/* * we timed out. Initiate RDMA LIF reset and indicate * error to caller. */ ionic_admin_timedout(aq); return -ETIMEDOUT;}
int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, enum ionic_admin_flags flags){ int rc, timo;
if (flags & IONIC_ADMIN_F_BUSYWAIT) { /* Spin */ rc = ionic_admin_busy_wait(wr); } else if (flags & IONIC_ADMIN_F_INTERRUPT) { /* * Interruptible sleep, 1s timeout * This is used for commands which are safe for the caller * to clean up without killing and resetting the adminq. */ timo = wait_for_completion_interruptible_timeout(&wr->work, HZ); if (timo > 0) rc = 0; else if (timo == 0) rc = -ETIMEDOUT; else rc = timo; } else { /* * Uninterruptible sleep * This is used for commands which are NOT safe for the * caller to clean up. Cleanup must be handled by the * adminq kill and reset process so that host memory is * not corrupted by the device. */ wait_for_completion(&wr->work); rc = 0; }
if (rc) { ibdev_warn(&dev->ibdev, "wait status %d\n", rc); ionic_admin_cancel(wr); } else if (wr->status == IONIC_ADMIN_KILLED) { ibdev_dbg(&dev->ibdev, "admin killed\n");
/* No error if admin already killed during teardown */ rc = (flags & IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV; } else if (ionic_v1_cqe_error(&wr->cqe)) { ibdev_warn(&dev->ibdev, "opcode %u error %u\n", wr->wqe.op, be32_to_cpu(wr->cqe.status_length)); rc = -EINVAL; } return rc;}
static int ionic_rdma_devcmd(struct ionic_ibdev *dev, struct ionic_admin_ctx *admin){ int rc;
rc = ionic_adminq_post_wait(dev->lif_cfg.lif, admin); if (rc) return rc;
return ionic_error_to_errno(admin->comp.comp.status);}
int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev){ struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_reset = { .opcode = IONIC_CMD_RDMA_RESET_LIF, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), }, };
return ionic_rdma_devcmd(dev, &admin);}
static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, struct ionic_queue *q, u32 qid, u32 cid, u16 opcode){ struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_queue = { .opcode = opcode, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), .qid_ver = cpu_to_le32(qid), .cid = cpu_to_le32(cid), .dbid = cpu_to_le16(dev->lif_cfg.dbid), .depth_log2 = q->depth_log2, .stride_log2 = q->stride_log2, .dma_addr = cpu_to_le64(q->dma), }, };
return ionic_rdma_devcmd(dev, &admin);}
static int ionic_rdma_cq_devcmd(struct ionic_vcq *vcq, struct ionic_queue *q, u32 *qid, u32 cid, u16 opcode, u8 udma_idx){ struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_queue = { .opcode = opcode, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), .qid_ver = cpu_to_le32(*qid), .cid = cpu_to_le32(cid), .dbid = cpu_to_le16(dev->lif_cfg.dbid), .depth_log2 = q->depth_log2, .stride_log2 = q->stride_log2, .dma_addr = cpu_to_le64(q->dma), }, }; int rc;
if (ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { struct ionic_admin_create_cq_resp resp_buf;
/* Indicates use of CQ create V2 response format. */ admin.cmd.rdma_queue.qid_ver = cpu_to_le32(IONIC_CREATE_CQ_CMD_V2_MAGIC); admin.cmd.rdma_queue.udma_idx = udma_idx;
rc = ionic_rdma_devcmd(dev, &admin); if (rc) return rc;
memcpy(&resp_buf, admin.comp.comp.cmd_data, sizeof(resp_buf)); *qid = le32_to_cpu(resp_buf.id); } else { rc = ionic_rdma_devcmd(dev, &admin); if (rc) return rc; }
return 0;}
static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context){ struct ionic_aq *aq = cq_context; unsigned long irqflags;
spin_lock_irqsave(&aq->lock, irqflags); aq->armed = false; if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) queue_work(ionic_evt_workq, &aq->work); spin_unlock_irqrestore(&aq->lock, irqflags);}
static void ionic_rdma_admincq_event(struct ib_event *event, void *cq_context){ struct ionic_aq *aq = cq_context;
ibdev_err(&aq->dev->ibdev, "admincq event %d\n", event->event);}
static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, int comp_vector){ struct ib_cq_init_attr attr = { .cqe = IONIC_AQ_DEPTH, .comp_vector = comp_vector, }; struct ionic_tbl_buf buf = {}; struct ionic_vcq *vcq; struct ionic_cq *cq; int rc;
vcq = kzalloc_obj(*vcq); if (!vcq) return ERR_PTR(-ENOMEM);
vcq->ibcq.device = &dev->ibdev; vcq->ibcq.comp_handler = ionic_rdma_admincq_comp; vcq->ibcq.event_handler = ionic_rdma_admincq_event; atomic_set(&vcq->ibcq.usecnt, 0);
vcq->udma_mask = 1; cq = &vcq->cq[0];
rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL, NULL, 0); if (rc) goto err_init;
rc = ionic_rdma_cq_devcmd(vcq, &cq->q, &cq->cqid, cq->eqid, IONIC_CMD_RDMA_CREATE_CQ, 0); if (rc) goto err_cmd;
rc = ionic_post_create_cq_cmd(cq, NULL, NULL); if (rc) goto err_cmd;
return vcq;
err_cmd: ionic_destroy_cq_common(dev, cq);err_init: kfree(vcq);
return ERR_PTR(rc);}
static void ionic_destroy_rdma_admincq(struct ionic_ibdev *dev, struct ionic_vcq *vcq){ ionic_pre_destroy_cq_cmd(dev, &vcq->cq[0]); ionic_destroy_cq_common(dev, &vcq->cq[0]); kfree(vcq);}
static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev, u32 aqid, u32 cqid){ struct ionic_aq *aq; int rc;
aq = kzalloc_obj(*aq); if (!aq) return ERR_PTR(-ENOMEM);
atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED); aq->dev = dev; aq->aqid = aqid; aq->cqid = cqid; spin_lock_init(&aq->lock);
rc = ionic_queue_init(&aq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH, ADMIN_WQE_STRIDE); if (rc) goto err_q;
ionic_queue_dbell_init(&aq->q, aq->aqid);
aq->q_wr = kzalloc_objs(*aq->q_wr, (u32)aq->q.mask + 1); if (!aq->q_wr) { rc = -ENOMEM; goto err_wr; }
INIT_LIST_HEAD(&aq->wr_prod); INIT_LIST_HEAD(&aq->wr_post);
INIT_WORK(&aq->work, ionic_admin_work); aq->armed = false;
return aq;
err_wr: ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev);err_q: kfree(aq);
return ERR_PTR(rc);}
static void __ionic_destroy_rdma_adminq(struct ionic_ibdev *dev, struct ionic_aq *aq){ kfree(aq->q_wr); ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev); kfree(aq);}
static struct ionic_aq *ionic_create_rdma_adminq(struct ionic_ibdev *dev, u32 aqid, u32 cqid){ struct ionic_aq *aq; int rc;
aq = __ionic_create_rdma_adminq(dev, aqid, cqid); if (IS_ERR(aq)) return aq;
rc = ionic_rdma_queue_devcmd(dev, &aq->q, aq->aqid, aq->cqid, IONIC_CMD_RDMA_CREATE_ADMINQ); if (rc) goto err_cmd;
return aq;
err_cmd: __ionic_destroy_rdma_adminq(dev, aq);
return ERR_PTR(rc);}
static void ionic_flush_qs(struct ionic_ibdev *dev){ struct ionic_qp *qp, *qp_tmp; struct ionic_cq *cq, *cq_tmp; LIST_HEAD(flush_list); unsigned long index;
WARN_ON(!irqs_disabled());
/* Flush qp send and recv */ xa_lock(&dev->qp_tbl); xa_for_each(&dev->qp_tbl, index, qp) { kref_get(&qp->qp_kref); list_add_tail(&qp->ibkill_flush_ent, &flush_list); } xa_unlock(&dev->qp_tbl);
list_for_each_entry_safe(qp, qp_tmp, &flush_list, ibkill_flush_ent) { ionic_flush_qp(dev, qp); kref_put(&qp->qp_kref, ionic_qp_complete); list_del(&qp->ibkill_flush_ent); }
/* Notify completions */ xa_lock(&dev->cq_tbl); xa_for_each(&dev->cq_tbl, index, cq) { kref_get(&cq->cq_kref); list_add_tail(&cq->ibkill_flush_ent, &flush_list); } xa_unlock(&dev->cq_tbl);
list_for_each_entry_safe(cq, cq_tmp, &flush_list, ibkill_flush_ent) { ionic_notify_flush_cq(cq); kref_put(&cq->cq_kref, ionic_cq_complete); list_del(&cq->ibkill_flush_ent); }}
static void ionic_kill_ibdev(struct ionic_ibdev *dev, bool fatal_path){ unsigned long irqflags; bool do_flush = false; int i;
/* Mark AQs for drain and flush the QPs while irq is disabled */ local_irq_save(irqflags);
/* Mark the admin queue, flushing at most once */ for (i = 0; i < dev->lif_cfg.aq_count; i++) { struct ionic_aq *aq = dev->aq_vec[i];
spin_lock(&aq->lock); if (atomic_read(&aq->admin_state) != IONIC_ADMIN_KILLED) { atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED); /* Flush incomplete admin commands */ ionic_admin_poll_locked(aq); do_flush = true; } spin_unlock(&aq->lock); }
if (do_flush) ionic_flush_qs(dev);
local_irq_restore(irqflags);
/* Post a fatal event if requested */ if (fatal_path) { struct ib_event ev;
ev.device = &dev->ibdev; ev.element.port_num = 1; ev.event = IB_EVENT_DEVICE_FATAL;
ib_dispatch_event(&ev); }
atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED);}
void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path){ enum ionic_admin_state old_state; unsigned long irqflags = 0; int i, rc;
if (!dev->aq_vec) return;
/* * Admin queues are transitioned from active to paused to killed state. * When in paused state, no new commands are issued to the device, * nor are any completed locally. After resetting the lif, it will be * safe to resume the rdma admin queues in the killed state. Commands * will not be issued to the device, but will complete locally with status * IONIC_ADMIN_KILLED. Handling completion will ensure that creating or * modifying resources fails, but destroying resources succeeds. * If there was a failure resetting the lif using this strategy, * then the state of the device is unknown. */ old_state = atomic_cmpxchg(&dev->admin_state, IONIC_ADMIN_ACTIVE, IONIC_ADMIN_PAUSED); if (old_state != IONIC_ADMIN_ACTIVE) return;
/* Pause all the AQs */ local_irq_save(irqflags); for (i = 0; i < dev->lif_cfg.aq_count; i++) { struct ionic_aq *aq = dev->aq_vec[i];
spin_lock(&aq->lock); /* pause rdma admin queues to reset lif */ if (atomic_read(&aq->admin_state) == IONIC_ADMIN_ACTIVE) atomic_set(&aq->admin_state, IONIC_ADMIN_PAUSED); spin_unlock(&aq->lock); } local_irq_restore(irqflags);
rc = ionic_rdma_reset_devcmd(dev); if (unlikely(rc)) { ibdev_err(&dev->ibdev, "failed to reset rdma %d\n", rc); ionic_request_rdma_reset(dev->lif_cfg.lif); }
ionic_kill_ibdev(dev, fatal_path);}
static void ionic_reset_work(struct work_struct *ws){ struct ionic_ibdev *dev = container_of(ws, struct ionic_ibdev, reset_work);
ionic_kill_rdma_admin(dev, true);}
static bool ionic_next_eqe(struct ionic_eq *eq, struct ionic_v1_eqe *eqe){ struct ionic_v1_eqe *qeqe; bool color;
qeqe = ionic_queue_at_prod(&eq->q); color = ionic_v1_eqe_color(qeqe);
/* cons is color for eq */ if (eq->q.cons != color) return false;
/* Prevent out-of-order reads of the EQE */ dma_rmb();
ibdev_dbg(&eq->dev->ibdev, "poll eq prod %u\n", eq->q.prod); print_hex_dump_debug("eqe ", DUMP_PREFIX_OFFSET, 16, 1, qeqe, BIT(eq->q.stride_log2), true); *eqe = *qeqe;
return true;}
static void ionic_cq_event(struct ionic_ibdev *dev, u32 cqid, u8 code){ unsigned long irqflags; struct ib_event ibev; struct ionic_cq *cq;
xa_lock_irqsave(&dev->cq_tbl, irqflags); cq = xa_load(&dev->cq_tbl, cqid); if (cq) kref_get(&cq->cq_kref); xa_unlock_irqrestore(&dev->cq_tbl, irqflags);
if (!cq) { ibdev_dbg(&dev->ibdev, "missing cqid %#x code %u\n", cqid, code); return; }
switch (code) { case IONIC_V1_EQE_CQ_NOTIFY: if (cq->vcq->ibcq.comp_handler) cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, cq->vcq->ibcq.cq_context); break;
case IONIC_V1_EQE_CQ_ERR: if (cq->vcq->ibcq.event_handler) { ibev.event = IB_EVENT_CQ_ERR; ibev.device = &dev->ibdev; ibev.element.cq = &cq->vcq->ibcq;
cq->vcq->ibcq.event_handler(&ibev, cq->vcq->ibcq.cq_context); } break;
default: ibdev_dbg(&dev->ibdev, "unrecognized cqid %#x code %u\n", cqid, code); break; }
kref_put(&cq->cq_kref, ionic_cq_complete);}
static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code){ unsigned long irqflags; struct ib_event ibev; struct ionic_qp *qp;
xa_lock_irqsave(&dev->qp_tbl, irqflags); qp = xa_load(&dev->qp_tbl, qpid); if (qp) kref_get(&qp->qp_kref); xa_unlock_irqrestore(&dev->qp_tbl, irqflags);
if (!qp) { ibdev_dbg(&dev->ibdev, "missing qpid %#x code %u\n", qpid, code); return; }
ibev.device = &dev->ibdev; ibev.element.qp = &qp->ibqp;
switch (code) { case IONIC_V1_EQE_SQ_DRAIN: ibev.event = IB_EVENT_SQ_DRAINED; break;
case IONIC_V1_EQE_QP_COMM_EST: ibev.event = IB_EVENT_COMM_EST; break;
case IONIC_V1_EQE_QP_LAST_WQE: ibev.event = IB_EVENT_QP_LAST_WQE_REACHED; break;
case IONIC_V1_EQE_QP_ERR: ibev.event = IB_EVENT_QP_FATAL; break;
case IONIC_V1_EQE_QP_ERR_REQUEST: ibev.event = IB_EVENT_QP_REQ_ERR; break;
case IONIC_V1_EQE_QP_ERR_ACCESS: ibev.event = IB_EVENT_QP_ACCESS_ERR; break;
default: ibdev_dbg(&dev->ibdev, "unrecognized qpid %#x code %u\n", qpid, code); goto out; }
if (qp->ibqp.event_handler) qp->ibqp.event_handler(&ibev, qp->ibqp.qp_context);
out: kref_put(&qp->qp_kref, ionic_qp_complete);}
static void ionic_srq_event(struct ionic_ibdev *dev, u32 srqid, u8 code){ unsigned long irqflags; struct ionic_srq *srq; struct ib_event ibev;
xa_lock_irqsave(&dev->srq_tbl, irqflags); srq = xa_load(&dev->srq_tbl, srqid); if (srq) kref_get(&srq->kref); xa_unlock_irqrestore(&dev->srq_tbl, irqflags);
if (!srq) { ibdev_dbg(&dev->ibdev, "missing srqid %#x code %u\n", srqid, code); return; }
ibev.device = &dev->ibdev; ibev.element.srq = &srq->ibsrq;
switch (code) { case IONIC_V1_EQE_SRQ_LIMIT_REACHED: ibev.event = IB_EVENT_SRQ_LIMIT_REACHED; break;
case IONIC_V1_EQE_SRQ_ERR: ibev.event = IB_EVENT_SRQ_ERR; break;
default: ibdev_dbg(&dev->ibdev, "unrecognized srqid %#x code %u\n", srqid, code); goto out; }
if (srq->ibsrq.event_handler) srq->ibsrq.event_handler(&ibev, srq->ibsrq.srq_context);
out: kref_put(&srq->kref, ionic_srq_complete);}
static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget){ struct ionic_ibdev *dev = eq->dev; struct ionic_v1_eqe eqe; u16 npolled = 0; u8 type, code; u32 evt, qid;
while (npolled < budget) { if (!ionic_next_eqe(eq, &eqe)) break;
ionic_queue_produce(&eq->q);
/* cons is color for eq */ eq->q.cons = ionic_color_wrap(eq->q.prod, eq->q.cons);
++npolled;
evt = ionic_v1_eqe_evt(&eqe); type = ionic_v1_eqe_evt_type(evt); code = ionic_v1_eqe_evt_code(evt); qid = ionic_v1_eqe_evt_qid(evt);
switch (type) { case IONIC_V1_EQE_TYPE_CQ: ionic_cq_event(dev, qid, code); break;
case IONIC_V1_EQE_TYPE_QP: ionic_qp_event(dev, qid, code); break;
case IONIC_V1_EQE_TYPE_SRQ: ionic_srq_event(dev, qid, code); break;
default: ibdev_dbg(&dev->ibdev, "unknown event %#x type %u\n", evt, type); } }
return npolled;}
static void ionic_poll_eq_work(struct work_struct *work){ struct ionic_eq *eq = container_of(work, struct ionic_eq, work); u32 npolled;
if (unlikely(!eq->enable) || WARN_ON(eq->armed)) return;
npolled = ionic_poll_eq(eq, IONIC_EQ_WORK_BUDGET); if (npolled == IONIC_EQ_WORK_BUDGET) { ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, npolled, 0); queue_work(ionic_evt_workq, &eq->work); } else { xchg(&eq->armed, 1); ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 0, IONIC_INTR_CRED_UNMASK); }}
static irqreturn_t ionic_poll_eq_isr(int irq, void *eqptr){ struct ionic_eq *eq = eqptr; int was_armed; u32 npolled;
was_armed = xchg(&eq->armed, 0);
if (unlikely(!eq->enable) || !was_armed) return IRQ_HANDLED;
npolled = ionic_poll_eq(eq, IONIC_EQ_ISR_BUDGET); if (npolled == IONIC_EQ_ISR_BUDGET) { ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, npolled, 0); queue_work(ionic_evt_workq, &eq->work); } else { xchg(&eq->armed, 1); ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 0, IONIC_INTR_CRED_UNMASK); }
return IRQ_HANDLED;}
static struct ionic_eq *ionic_create_eq(struct ionic_ibdev *dev, int eqid){ struct ionic_intr_info intr_obj = { }; struct ionic_eq *eq; int rc;
eq = kzalloc_obj(*eq); if (!eq) return ERR_PTR(-ENOMEM);
eq->dev = dev;
rc = ionic_queue_init(&eq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH, sizeof(struct ionic_v1_eqe)); if (rc) goto err_q;
eq->eqid = eqid;
eq->armed = true; eq->enable = false; INIT_WORK(&eq->work, ionic_poll_eq_work);
rc = ionic_intr_alloc(dev->lif_cfg.lif, &intr_obj); if (rc < 0) goto err_intr;
eq->irq = intr_obj.vector; eq->intr = intr_obj.index;
ionic_queue_dbell_init(&eq->q, eq->eqid);
/* cons is color for eq */ eq->q.cons = true;
snprintf(eq->name, sizeof(eq->name), "%s-%d-%d-eq", "ionr", dev->lif_cfg.lif_index, eq->eqid);
ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET); ionic_intr_mask_assert(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET); ionic_intr_coal_init(dev->lif_cfg.intr_ctrl, eq->intr, 0); ionic_intr_clean(dev->lif_cfg.intr_ctrl, eq->intr);
eq->enable = true;
rc = request_irq(eq->irq, ionic_poll_eq_isr, 0, eq->name, eq); if (rc) goto err_irq;
rc = ionic_rdma_queue_devcmd(dev, &eq->q, eq->eqid, eq->intr, IONIC_CMD_RDMA_CREATE_EQ); if (rc) goto err_cmd;
ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_CLEAR);
return eq;
err_cmd: eq->enable = false; free_irq(eq->irq, eq); flush_work(&eq->work);err_irq: ionic_intr_free(dev->lif_cfg.lif, eq->intr);err_intr: ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev);err_q: kfree(eq);
return ERR_PTR(rc);}
static void ionic_destroy_eq(struct ionic_eq *eq){ struct ionic_ibdev *dev = eq->dev;
eq->enable = false; free_irq(eq->irq, eq); flush_work(&eq->work);
ionic_intr_free(dev->lif_cfg.lif, eq->intr); ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev); kfree(eq);}
int ionic_create_rdma_admin(struct ionic_ibdev *dev){ int eq_i = 0, aq_i = 0, rc = 0; struct ionic_vcq *vcq; struct ionic_aq *aq; struct ionic_eq *eq;
dev->eq_vec = NULL; dev->aq_vec = NULL;
INIT_WORK(&dev->reset_work, ionic_reset_work); INIT_DELAYED_WORK(&dev->admin_dwork, ionic_admin_dwork); atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED);
if (dev->lif_cfg.aq_count > IONIC_AQ_COUNT) { ibdev_dbg(&dev->ibdev, "limiting adminq count to %d\n", IONIC_AQ_COUNT); dev->lif_cfg.aq_count = IONIC_AQ_COUNT; }
if (dev->lif_cfg.eq_count > IONIC_EQ_COUNT) { dev_dbg(&dev->ibdev.dev, "limiting eventq count to %d\n", IONIC_EQ_COUNT); dev->lif_cfg.eq_count = IONIC_EQ_COUNT; }
/* need at least two eq and one aq */ if (dev->lif_cfg.eq_count < IONIC_EQ_COUNT_MIN || dev->lif_cfg.aq_count < IONIC_AQ_COUNT_MIN) { rc = -EINVAL; goto out; }
dev->eq_vec = kmalloc_objs(*dev->eq_vec, dev->lif_cfg.eq_count); if (!dev->eq_vec) { rc = -ENOMEM; goto out; }
for (eq_i = 0; eq_i < dev->lif_cfg.eq_count; ++eq_i) { eq = ionic_create_eq(dev, eq_i + dev->lif_cfg.eq_base); if (IS_ERR(eq)) { rc = PTR_ERR(eq);
if (eq_i < IONIC_EQ_COUNT_MIN) { ibdev_err(&dev->ibdev, "fail create eq %pe\n", eq); goto out; }
/* ok, just fewer eq than device supports */ ibdev_dbg(&dev->ibdev, "eq count %d want %d rc %pe\n", eq_i, dev->lif_cfg.eq_count, eq);
rc = 0; break; }
dev->eq_vec[eq_i] = eq; }
dev->lif_cfg.eq_count = eq_i;
dev->aq_vec = kmalloc_objs(*dev->aq_vec, dev->lif_cfg.aq_count); if (!dev->aq_vec) { rc = -ENOMEM; goto out; }
/* Create one CQ per AQ */ for (aq_i = 0; aq_i < dev->lif_cfg.aq_count; ++aq_i) { vcq = ionic_create_rdma_admincq(dev, aq_i % eq_i); if (IS_ERR(vcq)) { rc = PTR_ERR(vcq);
if (!aq_i) { ibdev_err(&dev->ibdev, "failed to create acq %pe\n", vcq); goto out; }
/* ok, just fewer adminq than device supports */ ibdev_dbg(&dev->ibdev, "acq count %d want %d rc %pe\n", aq_i, dev->lif_cfg.aq_count, vcq); break; }
aq = ionic_create_rdma_adminq(dev, aq_i + dev->lif_cfg.aq_base, vcq->cq[0].cqid); if (IS_ERR(aq)) { /* Clean up the dangling CQ */ ionic_destroy_rdma_admincq(dev, vcq);
rc = PTR_ERR(aq);
if (!aq_i) { ibdev_err(&dev->ibdev, "failed to create aq %pe\n", aq); goto out; }
/* ok, just fewer adminq than device supports */ ibdev_dbg(&dev->ibdev, "aq count %d want %d rc %pe\n", aq_i, dev->lif_cfg.aq_count, aq); break; }
vcq->ibcq.cq_context = aq; aq->vcq = vcq;
atomic_set(&aq->admin_state, IONIC_ADMIN_ACTIVE); dev->aq_vec[aq_i] = aq; }
atomic_set(&dev->admin_state, IONIC_ADMIN_ACTIVE);out: dev->lif_cfg.eq_count = eq_i; dev->lif_cfg.aq_count = aq_i;
return rc;}
void ionic_destroy_rdma_admin(struct ionic_ibdev *dev){ struct ionic_vcq *vcq; struct ionic_aq *aq; struct ionic_eq *eq;
/* * Killing the admin before destroy makes sure all admin and * completions are flushed. admin_state = IONIC_ADMIN_KILLED * stops queueing up further works. */ cancel_delayed_work_sync(&dev->admin_dwork); cancel_work_sync(&dev->reset_work);
if (dev->aq_vec) { while (dev->lif_cfg.aq_count > 0) { aq = dev->aq_vec[--dev->lif_cfg.aq_count]; vcq = aq->vcq;
cancel_work_sync(&aq->work);
__ionic_destroy_rdma_adminq(dev, aq); if (vcq) ionic_destroy_rdma_admincq(dev, vcq); }
kfree(dev->aq_vec); }
if (dev->eq_vec) { while (dev->lif_cfg.eq_count > 0) { eq = dev->eq_vec[--dev->lif_cfg.eq_count]; ionic_destroy_eq(eq); }
kfree(dev->eq_vec); }}