Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
#include <linux/module.h>#include <linux/printk.h>#include <linux/pci.h>#include <linux/irq.h>#include <net/addrconf.h>#include <rdma/ib_addr.h>#include <rdma/ib_mad.h>
#include "ionic_ibdev.h"
#define DRIVER_DESCRIPTION "AMD Pensando RoCE HCA driver"#define DEVICE_DESCRIPTION "AMD Pensando RoCE HCA"
MODULE_AUTHOR("Allen Hubbe <allen.hubbe@amd.com>");MODULE_DESCRIPTION(DRIVER_DESCRIPTION);MODULE_LICENSE("GPL");MODULE_IMPORT_NS("NET_IONIC");
static int ionic_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *udata){ struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); struct net_device *ndev; int err;
err = ib_no_udata_io(udata); if (err) return err;
ndev = ib_device_get_netdev(ibdev, 1); addrconf_ifid_eui48((u8 *)&attr->sys_image_guid, ndev); dev_put(ndev); attr->max_mr_size = dev->lif_cfg.npts_per_lif * PAGE_SIZE / 2; attr->page_size_cap = dev->lif_cfg.page_size_supported;
attr->vendor_id = to_pci_dev(dev->lif_cfg.hwdev)->vendor; attr->vendor_part_id = to_pci_dev(dev->lif_cfg.hwdev)->device;
attr->hw_ver = ionic_lif_asic_rev(dev->lif_cfg.lif); attr->fw_ver = 0; attr->max_qp = dev->lif_cfg.qp_count; attr->max_qp_wr = IONIC_MAX_DEPTH; attr->device_cap_flags = IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_MEM_WINDOW_TYPE_2B | 0; attr->max_send_sge = min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), IONIC_SPEC_HIGH); attr->max_recv_sge = min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), IONIC_SPEC_HIGH); attr->max_sge_rd = attr->max_send_sge; attr->max_cq = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count; attr->max_cqe = IONIC_MAX_CQ_DEPTH - IONIC_CQ_GRACE; attr->max_mr = dev->lif_cfg.nmrs_per_lif; attr->max_pd = IONIC_MAX_PD; attr->max_qp_rd_atom = IONIC_MAX_RD_ATOM; attr->max_ee_rd_atom = 0; attr->max_res_rd_atom = IONIC_MAX_RD_ATOM; attr->max_qp_init_rd_atom = IONIC_MAX_RD_ATOM; attr->max_ee_init_rd_atom = 0; attr->atomic_cap = IB_ATOMIC_GLOB; attr->masked_atomic_cap = IB_ATOMIC_GLOB; attr->max_mw = dev->lif_cfg.nmrs_per_lif; attr->max_mcast_grp = 0; attr->max_mcast_qp_attach = 0; attr->max_ah = dev->lif_cfg.nahs_per_lif; attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; attr->max_pkeys = IONIC_PKEY_TBL_LEN; if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) { attr->max_srq = dev->lif_cfg.srq_count; attr->max_srq_wr = IONIC_MAX_SRQ_DEPTH; attr->max_srq_sge = IONIC_MAX_SRQ_SGES; }
return 0;}
static int ionic_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *attr){ struct net_device *ndev;
if (port != 1) return -EINVAL;
ndev = ib_device_get_netdev(ibdev, port); if (!ndev) return -ENODEV;
if (netif_running(ndev) && netif_carrier_ok(ndev)) { attr->state = IB_PORT_ACTIVE; attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; } else if (netif_running(ndev)) { attr->state = IB_PORT_DOWN; attr->phys_state = IB_PORT_PHYS_STATE_POLLING; } else { attr->state = IB_PORT_DOWN; attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; }
attr->max_mtu = iboe_get_mtu(ndev->max_mtu); attr->active_mtu = min(attr->max_mtu, iboe_get_mtu(ndev->mtu)); attr->gid_tbl_len = IONIC_GID_TBL_LEN; attr->ip_gids = true; attr->port_cap_flags = 0; attr->max_msg_sz = 0x80000000; attr->pkey_tbl_len = IONIC_PKEY_TBL_LEN; attr->max_vl_num = 1; attr->subnet_prefix = 0xfe80000000000000ull;
dev_put(ndev);
return ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);}
static enum rdma_link_layer ionic_get_link_layer(struct ib_device *ibdev, u32 port){ return IB_LINK_LAYER_ETHERNET;}
static int ionic_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey){ if (port != 1) return -EINVAL;
if (index != 0) return -EINVAL;
*pkey = IB_DEFAULT_PKEY_FULL;
return 0;}
static int ionic_modify_device(struct ib_device *ibdev, int mask, struct ib_device_modify *attr){ struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);
if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) return -EOPNOTSUPP;
if (mask & IB_DEVICE_MODIFY_NODE_DESC) memcpy(dev->ibdev.node_desc, attr->node_desc, IB_DEVICE_NODE_DESC_MAX);
return 0;}
static int ionic_get_port_immutable(struct ib_device *ibdev, u32 port, struct ib_port_immutable *attr){ if (port != 1) return -EINVAL;
attr->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
attr->pkey_tbl_len = IONIC_PKEY_TBL_LEN; attr->gid_tbl_len = IONIC_GID_TBL_LEN; attr->max_mad_size = IB_MGMT_MAD_SIZE;
return 0;}
static void ionic_get_dev_fw_str(struct ib_device *ibdev, char *str){ struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);
ionic_lif_fw_version(dev->lif_cfg.lif, str, IB_FW_VERSION_NAME_MAX);}
static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, char *buf){ struct ionic_ibdev *dev = rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev);
return sysfs_emit(buf, "0x%x\n", ionic_lif_asic_rev(dev->lif_cfg.lif));}static DEVICE_ATTR_RO(hw_rev);
static ssize_t hca_type_show(struct device *device, struct device_attribute *attr, char *buf){ struct ionic_ibdev *dev = rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev);
return sysfs_emit(buf, "%.64s\n", dev->ibdev.node_desc);}static DEVICE_ATTR_RO(hca_type);
static struct attribute *ionic_rdma_attributes[] = { &dev_attr_hw_rev.attr, &dev_attr_hca_type.attr, NULL};
static const struct attribute_group ionic_rdma_attr_group = { .attrs = ionic_rdma_attributes,};
static void ionic_disassociate_ucontext(struct ib_ucontext *ibcontext){ /* * Dummy define disassociate_ucontext so that it does not * wait for user context before cleaning up hw resources. */}
static const struct ib_device_ops ionic_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_IONIC, .uverbs_abi_ver = IONIC_ABI_VERSION, .uverbs_robust_udata = true,
.alloc_ucontext = ionic_alloc_ucontext, .dealloc_ucontext = ionic_dealloc_ucontext, .mmap = ionic_mmap, .mmap_free = ionic_mmap_free, .alloc_pd = ionic_alloc_pd, .dealloc_pd = ionic_dealloc_pd, .create_ah = ionic_create_ah, .query_ah = ionic_query_ah, .destroy_ah = ionic_destroy_ah, .create_user_ah = ionic_create_ah, .get_dma_mr = ionic_get_dma_mr, .reg_user_mr = ionic_reg_user_mr, .reg_user_mr_dmabuf = ionic_reg_user_mr_dmabuf, .dereg_mr = ionic_dereg_mr, .alloc_mr = ionic_alloc_mr, .map_mr_sg = ionic_map_mr_sg, .alloc_mw = ionic_alloc_mw, .dealloc_mw = ionic_dealloc_mw, .create_cq = ionic_create_cq, .destroy_cq = ionic_destroy_cq, .create_qp = ionic_create_qp, .modify_qp = ionic_modify_qp, .query_qp = ionic_query_qp, .destroy_qp = ionic_destroy_qp,
.post_send = ionic_post_send, .post_recv = ionic_post_recv, .poll_cq = ionic_poll_cq, .req_notify_cq = ionic_req_notify_cq,
.query_device = ionic_query_device, .query_port = ionic_query_port, .get_link_layer = ionic_get_link_layer, .query_pkey = ionic_query_pkey, .modify_device = ionic_modify_device, .get_port_immutable = ionic_get_port_immutable, .get_dev_fw_str = ionic_get_dev_fw_str, .device_group = &ionic_rdma_attr_group, .disassociate_ucontext = ionic_disassociate_ucontext,
INIT_RDMA_OBJ_SIZE(ib_ucontext, ionic_ctx, ibctx), INIT_RDMA_OBJ_SIZE(ib_pd, ionic_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ah, ionic_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, ionic_vcq, ibcq), INIT_RDMA_OBJ_SIZE(ib_qp, ionic_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_mw, ionic_mr, ibmw),};
static const struct ib_device_ops ionic_srq_ops = { .create_srq = ionic_create_srq, .modify_srq = ionic_modify_srq, .query_srq = ionic_query_srq, .destroy_srq = ionic_destroy_srq,
INIT_RDMA_OBJ_SIZE(ib_srq, ionic_srq, ibsrq),};
static void ionic_init_resids(struct ionic_ibdev *dev){ ionic_resid_init(&dev->inuse_cqid, dev->lif_cfg.cq_count); dev->half_cqid_udma_shift = order_base_2(dev->lif_cfg.cq_count / dev->lif_cfg.udma_count); ionic_resid_init(&dev->inuse_pdid, IONIC_MAX_PD); ionic_resid_init(&dev->inuse_ahid, dev->lif_cfg.nahs_per_lif); ionic_resid_init(&dev->inuse_mrid, dev->lif_cfg.nmrs_per_lif); /* skip reserved lkey */ dev->next_mrkey = 1; ionic_resid_init(&dev->inuse_qpid, dev->lif_cfg.qp_count); /* skip reserved SMI and GSI qpids */ dev->half_qpid_udma_shift = order_base_2(dev->lif_cfg.qp_count / dev->lif_cfg.udma_count); ionic_resid_init(&dev->inuse_dbid, dev->lif_cfg.dbid_count);}
static void ionic_destroy_resids(struct ionic_ibdev *dev){ ionic_resid_destroy(&dev->inuse_cqid); ionic_resid_destroy(&dev->inuse_pdid); ionic_resid_destroy(&dev->inuse_ahid); ionic_resid_destroy(&dev->inuse_mrid); ionic_resid_destroy(&dev->inuse_qpid); ionic_resid_destroy(&dev->inuse_dbid);}
static void ionic_destroy_ibdev(struct ionic_ibdev *dev){ ionic_kill_rdma_admin(dev, false); ib_unregister_device(&dev->ibdev); ionic_stats_cleanup(dev); ionic_destroy_rdma_admin(dev); ionic_destroy_resids(dev); WARN_ON(!xa_empty(&dev->qp_tbl)); xa_destroy(&dev->qp_tbl); WARN_ON(!xa_empty(&dev->cq_tbl)); xa_destroy(&dev->cq_tbl); WARN_ON(!xa_empty(&dev->srq_tbl)); xa_destroy(&dev->srq_tbl); ib_dealloc_device(&dev->ibdev);}
static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev){ struct ib_device *ibdev; struct ionic_ibdev *dev; struct net_device *ndev; int rc;
dev = ib_alloc_device(ionic_ibdev, ibdev); if (!dev) return ERR_PTR(-EINVAL);
ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg);
xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ); xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ); xa_init_flags(&dev->srq_tbl, XA_FLAGS_LOCK_IRQ);
ionic_init_resids(dev);
rc = ionic_rdma_reset_devcmd(dev); if (rc) goto err_reset;
rc = ionic_create_rdma_admin(dev); if (rc) goto err_admin;
ibdev = &dev->ibdev; ibdev->dev.parent = dev->lif_cfg.hwdev;
strscpy(ibdev->name, "ionic_%d", IB_DEVICE_NAME_MAX); strscpy(ibdev->node_desc, DEVICE_DESCRIPTION, IB_DEVICE_NODE_DESC_MAX);
ibdev->node_type = RDMA_NODE_IB_CA; ibdev->phys_port_cnt = 1;
/* the first two eq are reserved for async events */ ibdev->num_comp_vectors = dev->lif_cfg.eq_count - 2;
ndev = ionic_lif_netdev(ionic_adev->lif); addrconf_ifid_eui48((u8 *)&ibdev->node_guid, ndev); rc = ib_device_set_netdev(ibdev, ndev, 1); /* ionic_lif_netdev() returns ndev with refcount held */ dev_put(ndev); if (rc) goto err_admin;
ib_set_device_ops(&dev->ibdev, &ionic_dev_ops); if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) ib_set_device_ops(&dev->ibdev, &ionic_srq_ops);
ionic_stats_init(dev);
rc = ib_register_device(ibdev, "ionic_%d", ibdev->dev.parent); if (rc) goto err_register;
return dev;
err_register: ionic_stats_cleanup(dev);err_admin: ionic_kill_rdma_admin(dev, false); ionic_destroy_rdma_admin(dev);err_reset: ionic_destroy_resids(dev); xa_destroy(&dev->qp_tbl); xa_destroy(&dev->cq_tbl); xa_destroy(&dev->srq_tbl); ib_dealloc_device(&dev->ibdev);
return ERR_PTR(rc);}
static int ionic_aux_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id){ struct ionic_aux_dev *ionic_adev; struct ionic_ibdev *dev;
ionic_adev = container_of(adev, struct ionic_aux_dev, adev); dev = ionic_create_ibdev(ionic_adev); if (IS_ERR(dev)) return dev_err_probe(&adev->dev, PTR_ERR(dev), "Failed to register ibdev\n");
auxiliary_set_drvdata(adev, dev); ibdev_dbg(&dev->ibdev, "registered\n");
return 0;}
static void ionic_aux_remove(struct auxiliary_device *adev){ struct ionic_ibdev *dev = auxiliary_get_drvdata(adev);
dev_dbg(&adev->dev, "unregister ibdev\n"); ionic_destroy_ibdev(dev); dev_dbg(&adev->dev, "unregistered\n");}
static const struct auxiliary_device_id ionic_aux_id_table[] = { { .name = "ionic.rdma", }, {},};
MODULE_DEVICE_TABLE(auxiliary, ionic_aux_id_table);
static struct auxiliary_driver ionic_aux_r_driver = { .name = "rdma", .probe = ionic_aux_probe, .remove = ionic_aux_remove, .id_table = ionic_aux_id_table,};
static int __init ionic_mod_init(void){ int rc;
ionic_evt_workq = create_workqueue(KBUILD_MODNAME "-evt"); if (!ionic_evt_workq) return -ENOMEM;
rc = auxiliary_driver_register(&ionic_aux_r_driver); if (rc) goto err_aux;
return 0;
err_aux: destroy_workqueue(ionic_evt_workq);
return rc;}
static void __exit ionic_mod_exit(void){ auxiliary_driver_unregister(&ionic_aux_r_driver); destroy_workqueue(ionic_evt_workq);}
module_init(ionic_mod_init);module_exit(ionic_mod_exit);