Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573/* SPDX-License-Identifier: GPL-2.0 *//* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
#ifndef _IONIC_IBDEV_H_#define _IONIC_IBDEV_H_
#include <rdma/ib_verbs.h>#include <rdma/ib_pack.h>#include <rdma/iter.h>#include <rdma/uverbs_ioctl.h>
#include <rdma/ionic-abi.h>#include <ionic_api.h>#include <ionic_regs.h>
#include "ionic_fw.h"#include "ionic_queue.h"#include "ionic_res.h"
#include "ionic_lif_cfg.h"
/* Config knobs */#define IONIC_EQ_DEPTH 511#define IONIC_EQ_COUNT 32#define IONIC_AQ_DEPTH 63#define IONIC_AQ_COUNT 4#define IONIC_EQ_ISR_BUDGET 10#define IONIC_EQ_WORK_BUDGET 1000#define IONIC_MAX_RD_ATOM 16#define IONIC_PKEY_TBL_LEN 1#define IONIC_GID_TBL_LEN 256
#define IONIC_MAX_QPID 0xffffff#define IONIC_SPEC_HIGH 8#define IONIC_MAX_PD 1024#define IONIC_SQCMB_ORDER 5#define IONIC_RQCMB_ORDER 0
#define IONIC_MAX_SRQ_SGES 2#define IONIC_MAX_SRQ_LIMIT 0xffff#define IONIC_MAX_SRQ_DEPTH 0xffff
#define IONIC_META_LAST ((void *)1ul)#define IONIC_META_POSTED ((void *)2ul)
#define IONIC_CQ_GRACE 100
#define IONIC_ROCE_UDP_SPORT 28272#define IONIC_DMA_LKEY 0#define IONIC_DMA_RKEY IONIC_DMA_LKEY
#define IONIC_CMB_SUPPORTED \ (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ IONIC_CMB_WC | IONIC_CMB_UC)
/* resource is not reserved on the device, indicated in tbl_order */#define IONIC_RES_INVALID -1
struct ionic_aq;struct ionic_cq;struct ionic_eq;struct ionic_vcq;
enum ionic_admin_state { IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ IONIC_ADMIN_KILLED, /* not submitting, locally completed */};
enum ionic_admin_flags { IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */};
enum ionic_mmap_flag { IONIC_MMAP_WC = BIT(0), IONIC_MMAP_PHC = BIT(1),};
struct ionic_mmap_entry { struct rdma_user_mmap_entry rdma_entry; unsigned long size; unsigned long pfn; u8 mmap_flags;};
struct ionic_ibdev { struct ib_device ibdev;
struct ionic_lif_cfg lif_cfg;
struct xarray qp_tbl; struct xarray cq_tbl; struct xarray srq_tbl;
struct ionic_resid_bits inuse_dbid; struct ionic_resid_bits inuse_pdid; struct ionic_resid_bits inuse_ahid; struct ionic_resid_bits inuse_mrid; struct ionic_resid_bits inuse_qpid; struct ionic_resid_bits inuse_cqid;
u8 half_cqid_udma_shift; u8 half_qpid_udma_shift; u8 next_qpid_udma_idx; u8 next_mrkey;
struct work_struct reset_work; bool reset_posted; u32 reset_cnt;
struct delayed_work admin_dwork; struct ionic_aq **aq_vec; atomic_t admin_state;
struct ionic_eq **eq_vec;
struct ionic_v1_stat *hw_stats; void *hw_stats_buf; struct rdma_stat_desc *hw_stats_hdrs; struct ionic_counter_stats *counter_stats; int hw_stats_count;};
struct ionic_eq { struct ionic_ibdev *dev;
u32 eqid; u32 intr;
struct ionic_queue q;
int armed; bool enable;
struct work_struct work;
int irq; char name[32];};
struct ionic_admin_wr { struct completion work; struct list_head aq_ent; struct ionic_v1_admin_wqe wqe; struct ionic_v1_cqe cqe; struct ionic_aq *aq; int status;};
struct ionic_admin_wr_q { struct ionic_admin_wr *wr; int wqe_strides;};
struct ionic_aq { struct ionic_ibdev *dev; struct ionic_vcq *vcq;
struct work_struct work;
atomic_t admin_state; unsigned long stamp; bool armed;
u32 aqid; u32 cqid;
spinlock_t lock; /* for posting */ struct ionic_queue q; struct ionic_admin_wr_q *q_wr; struct list_head wr_prod; struct list_head wr_post;};
struct ionic_ctx { struct ib_ucontext ibctx; u32 dbid; struct rdma_user_mmap_entry *mmap_dbell; struct rdma_user_mmap_entry *mmap_phc;};
struct ionic_tbl_buf { u32 tbl_limit; u32 tbl_pages; size_t tbl_size; __le64 *tbl_buf; dma_addr_t tbl_dma; u8 page_size_log2;};
struct ionic_pd { struct ib_pd ibpd;
u32 pdid; u32 flags;};
struct ionic_cq { struct ionic_vcq *vcq;
u32 cqid; u32 eqid;
spinlock_t lock; /* for polling */ struct list_head poll_sq; bool flush; struct list_head flush_sq; struct list_head flush_rq; struct list_head ibkill_flush_ent;
struct ionic_queue q; bool color; int credit; u16 arm_any_prod; u16 arm_sol_prod;
struct kref cq_kref; struct completion cq_rel_comp;
/* infrequently accessed, keep at end */ struct ib_umem *umem;};
struct ionic_vcq { struct ib_cq ibcq; struct ionic_cq cq[2]; u8 udma_mask; u8 poll_idx;};
struct ionic_sq_meta { u64 wrid; u32 len; u16 seq; u8 ibop; u8 ibsts; u8 remote:1; u8 signal:1; u8 local_comp:1;};
struct ionic_rq_meta { struct ionic_rq_meta *next; u64 wrid;};
struct ionic_rq { struct ionic_queue q; u32 qid; u32 cqid; struct list_head cq_flush; spinlock_t lock; /* for posting and polling */
phys_addr_t cmb_addr; struct rdma_user_mmap_entry *mmap_cmb;
struct ionic_rq_meta *meta; struct ionic_rq_meta *meta_head; struct ib_umem *umem;
int spec; int cmb_order; u32 cmb_pgid; u16 old_prod; u8 cmb; bool flush;};
struct ionic_srq { struct ib_srq ibsrq; struct ionic_rq rq; struct kref kref; struct completion rel_comp; u16 srq_limit; u8 udma_idx;};
struct ionic_qp { struct ib_qp ibqp; enum ib_qp_state state;
u32 qpid; u32 ahid; u32 sq_cqid; u8 udma_idx; u8 has_ah:1; u8 has_sq:1; u8 sig_all:1;
struct list_head qp_list_counter;
struct list_head cq_poll_sq; struct list_head cq_flush_sq; struct list_head ibkill_flush_ent;
spinlock_t sq_lock; /* for posting and polling */ struct ionic_queue sq; struct ionic_sq_meta *sq_meta; u16 *sq_msn_idx; int sq_spec; u16 sq_old_prod; u16 sq_msn_prod; u16 sq_msn_cons; u8 sq_cmb; bool sq_flush; bool sq_flush_rcvd;
struct ionic_rq rq;
struct kref qp_kref; struct completion qp_rel_comp;
/* infrequently accessed, keep at end */ int sgid_index; int sq_cmb_order; u32 sq_cmb_pgid; phys_addr_t sq_cmb_addr; struct rdma_user_mmap_entry *mmap_sq_cmb;
struct ib_umem *sq_umem;
int dcqcn_profile;
struct ib_ud_header *hdr; struct ionic_srq *srq;};
struct ionic_ah { struct ib_ah ibah; u32 ahid; int sgid_index; struct ib_ud_header hdr;};
struct ionic_mr { union { struct ib_mr ibmr; struct ib_mw ibmw; };
u32 mrid; int flags;
struct ib_umem *umem; struct ionic_tbl_buf buf; bool created;};
struct ionic_counter_stats { int queue_stats_count; struct ionic_v1_stat *hdr; struct rdma_stat_desc *stats_hdrs; struct ida counter_ida;};
struct ionic_rdma_counter { struct rdma_counter rdma_counter; void *vals; struct list_head qp_list;};
static inline struct ionic_rdma_counter *to_ionic_rdma_counter(struct rdma_counter *counter){ return container_of(counter, struct ionic_rdma_counter, rdma_counter);}
static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev){ return container_of(ibdev, struct ionic_ibdev, ibdev);}
static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx){ return container_of(ibctx, struct ionic_ctx, ibctx);}
static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj){ if (!uobj) return NULL;
if (!uobj->context) return NULL;
return to_ionic_ctx(uobj->context);}
static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd){ return container_of(ibpd, struct ionic_pd, ibpd);}
static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr){ return container_of(ibmr, struct ionic_mr, ibmr);}
static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw){ return container_of(ibmw, struct ionic_mr, ibmw);}
static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq){ return container_of(ibcq, struct ionic_vcq, ibcq);}
static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, uint8_t udma_idx){ return &to_ionic_vcq(ibcq)->cq[udma_idx];}
static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp){ return container_of(ibqp, struct ionic_qp, ibqp);}
static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq){ return container_of(ibsrq, struct ionic_srq, ibsrq);}
static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah){ return container_of(ibah, struct ionic_ah, ibah);}
static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, struct ionic_ctx *ctx){ if (!ctx) return dev->lif_cfg.dbid;
return ctx->dbid;}
static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, struct ib_uobject *uobj){ return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));}
static inline bool ionic_ibop_is_local(enum ib_wr_opcode op){ return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;}
static inline void ionic_qp_complete(struct kref *kref){ struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);
complete(&qp->qp_rel_comp);}
static inline void ionic_cq_complete(struct kref *kref){ struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);
complete(&cq->cq_rel_comp);}
static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev, enum ionic_lif_rdma_alloc_qid qtype){ return dev->lif_cfg.alloc_qid_cap & qtype;}
static inline void ionic_srq_complete(struct kref *kref){ struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref);
complete(&srq->rel_comp);}
/* ionic_admin.c */extern struct workqueue_struct *ionic_evt_workq;void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, enum ionic_admin_flags);
int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);
int ionic_create_rdma_admin(struct ionic_ibdev *dev);void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);
/* ionic_controlpath.c */int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_tbl_buf *buf, const struct ib_cq_init_attr *attr, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, int udma_idx);int ionic_post_create_cq_cmd(struct ionic_cq *cq, struct ib_udata *udata, __u32 *resp_cqid);void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq);void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);void ionic_notify_flush_cq(struct ionic_cq *cq);
int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata);int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 addr, int access, struct ib_dmah *dmah, struct ib_udata *udata);struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, u64 length, u64 addr, int fd, int access, struct ib_dmah *dmah, struct uverbs_attr_bundle *attrs);int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, u32 max_sg);int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset);int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);int ionic_dealloc_mw(struct ib_mw *ibmw);int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs);int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, struct ib_udata *udata);int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_udata *udata);int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_qp_init_attr *init_attr);int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, struct ib_udata *udata);int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
/* ionic_datapath.c */int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad);int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad);int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
/* ionic_hw_stats.c */void ionic_stats_init(struct ionic_ibdev *dev);void ionic_stats_cleanup(struct ionic_ibdev *dev);
/* ionic_pgtbl.c */__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);int ionic_pgtbl_init(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf, struct ib_umem *umem, dma_addr_t dma, int limit, u64 page_size);void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);#endif /* _IONIC_IBDEV_H_ */