Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784// SPDX-License-Identifier: GPL-2.0/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */#include <linux/init.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/pci.h>#include <linux/device.h>#include <linux/sched/task.h>#include <linux/io-64-nonatomic-lo-hi.h>#include <linux/cdev.h>#include <linux/fs.h>#include <linux/poll.h>#include <linux/iommu.h>#include <linux/highmem.h>#include <uapi/linux/idxd.h>#include <linux/xarray.h>#include "registers.h"#include "idxd.h"
struct idxd_cdev_context { const char *name; dev_t devt; struct ida minor_ida;};
/* * Since user file names are global in DSA devices, define their ida's as * global to avoid conflict file names. */static DEFINE_IDA(file_ida);
/* * ictx is an array based off of accelerator types. enum idxd_type * is used as index */static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { { .name = "dsa" }, { .name = "iax" }};
struct idxd_user_context { struct idxd_wq *wq; struct task_struct *task; unsigned int pasid; struct mm_struct *mm; unsigned int flags; struct iommu_sva *sva; struct idxd_dev idxd_dev; u64 counters[COUNTER_MAX]; int id; pid_t pid;};
static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
static inline struct idxd_user_context *dev_to_uctx(struct device *dev){ struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return container_of(idxd_dev, struct idxd_user_context, idxd_dev);}
static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf){ struct idxd_user_context *ctx = dev_to_uctx(dev);
return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);}static DEVICE_ATTR_RO(cr_faults);
static ssize_t cr_fault_failures_show(struct device *dev, struct device_attribute *attr, char *buf){ struct idxd_user_context *ctx = dev_to_uctx(dev);
return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);}static DEVICE_ATTR_RO(cr_fault_failures);
static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf){ struct idxd_user_context *ctx = dev_to_uctx(dev);
return sysfs_emit(buf, "%u\n", ctx->pid);}static DEVICE_ATTR_RO(pid);
static struct attribute *cdev_file_attributes[] = { &dev_attr_cr_faults.attr, &dev_attr_cr_fault_failures.attr, &dev_attr_pid.attr, NULL};
static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n){ struct device *dev = container_of(kobj, typeof(*dev), kobj); struct idxd_user_context *ctx = dev_to_uctx(dev); struct idxd_wq *wq = ctx->wq;
if (!wq_pasid_enabled(wq)) return 0;
return a->mode;}
static const struct attribute_group cdev_file_attribute_group = { .attrs = cdev_file_attributes, .is_visible = cdev_file_attr_visible,};
static const struct attribute_group *cdev_file_attribute_groups[] = { &cdev_file_attribute_group, NULL};
static void idxd_file_dev_release(struct device *dev){ struct idxd_user_context *ctx = dev_to_uctx(dev); struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; int rc;
ida_free(&file_ida, ctx->id);
/* Wait for in-flight operations to complete. */ if (wq_shared(wq)) { idxd_device_drain_pasid(idxd, ctx->pasid); } else { if (device_user_pasid_enabled(idxd)) { /* The wq disable in the disable pasid function will drain the wq */ rc = idxd_wq_disable_pasid(wq); if (rc < 0) dev_err(dev, "wq disable pasid failed.\n"); } else { idxd_wq_drain(wq); } }
if (ctx->sva) { idxd_cdev_evl_drain_pasid(wq, ctx->pasid); iommu_sva_unbind_device(ctx->sva); idxd_xa_pasid_remove(ctx); } kfree(ctx); mutex_lock(&wq->wq_lock); idxd_wq_put(wq); mutex_unlock(&wq->wq_lock);}
static const struct device_type idxd_cdev_file_type = { .name = "idxd_file", .release = idxd_file_dev_release, .groups = cdev_file_attribute_groups,};
static void idxd_cdev_dev_release(struct device *dev){ struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
kfree(idxd_cdev);}
static const struct device_type idxd_cdev_device_type = { .name = "idxd_cdev", .release = idxd_cdev_dev_release,};
static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode){ struct cdev *cdev = inode->i_cdev;
return container_of(cdev, struct idxd_cdev, cdev);}
static inline struct idxd_wq *inode_wq(struct inode *inode){ struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
return idxd_cdev->wq;}
static void idxd_xa_pasid_remove(struct idxd_user_context *ctx){ struct idxd_wq *wq = ctx->wq; void *ptr;
mutex_lock(&wq->uc_lock); ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); if (ptr != (void *)ctx) dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", ctx->pasid); mutex_unlock(&wq->uc_lock);}
void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index){ struct idxd_user_context *ctx;
if (index >= COUNTER_MAX) return;
mutex_lock(&wq->uc_lock); ctx = xa_load(&wq->upasid_xa, pasid); if (!ctx) { mutex_unlock(&wq->uc_lock); return; } ctx->counters[index]++; mutex_unlock(&wq->uc_lock);}
static int idxd_cdev_open(struct inode *inode, struct file *filp){ struct idxd_user_context *ctx; struct idxd_device *idxd; struct idxd_wq *wq; struct device *dev, *fdev; int rc = 0; struct iommu_sva *sva = NULL; unsigned int pasid; struct idxd_cdev *idxd_cdev;
wq = inode_wq(inode); idxd = wq->idxd; dev = &idxd->pdev->dev;
dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
ctx = kzalloc_obj(*ctx); if (!ctx) return -ENOMEM;
mutex_lock(&wq->wq_lock);
if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { rc = -EBUSY; goto failed; }
ctx->wq = wq; filp->private_data = ctx; ctx->pid = current->pid;
if (device_user_pasid_enabled(idxd)) { sva = iommu_sva_bind_device(dev, current->mm); if (IS_ERR(sva)) { rc = PTR_ERR(sva); dev_err(dev, "pasid allocation failed: %d\n", rc); goto failed; }
pasid = iommu_sva_get_pasid(sva); if (pasid == IOMMU_PASID_INVALID) { rc = -EINVAL; goto failed_get_pasid; }
ctx->sva = sva; ctx->pasid = pasid; ctx->mm = current->mm;
mutex_lock(&wq->uc_lock); rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); mutex_unlock(&wq->uc_lock); if (rc < 0) dev_warn(dev, "PASID entry already exist in xarray.\n");
if (wq_dedicated(wq)) { rc = idxd_wq_set_pasid(wq, pasid); if (rc < 0) { dev_err(dev, "wq set pasid failed: %d\n", rc); goto failed_set_pasid; } } }
idxd_cdev = wq->idxd_cdev; ctx->id = ida_alloc(&file_ida, GFP_KERNEL); if (ctx->id < 0) { dev_warn(dev, "ida alloc failure\n"); goto failed_ida; } ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; fdev = user_ctx_dev(ctx); device_initialize(fdev); fdev->parent = cdev_dev(idxd_cdev); fdev->bus = &dsa_bus_type; fdev->type = &idxd_cdev_file_type;
rc = dev_set_name(fdev, "file%d", ctx->id); if (rc < 0) { dev_warn(dev, "set name failure\n"); goto failed_dev_name; }
rc = device_add(fdev); if (rc < 0) { dev_warn(dev, "file device add failure\n"); goto failed_dev_add; }
idxd_wq_get(wq); mutex_unlock(&wq->wq_lock); return 0;
failed_dev_add:failed_dev_name: put_device(fdev);failed_ida:failed_set_pasid: if (device_user_pasid_enabled(idxd)) idxd_xa_pasid_remove(ctx);failed_get_pasid: if (device_user_pasid_enabled(idxd) && !IS_ERR_OR_NULL(sva)) iommu_sva_unbind_device(sva);failed: mutex_unlock(&wq->wq_lock); kfree(ctx); return rc;}
static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid){ struct idxd_device *idxd = wq->idxd; struct idxd_evl *evl = idxd->evl; union evl_status_reg status; u16 h, t, size; int ent_size = evl_ent_size(idxd); struct __evl_entry *entry_head;
if (!evl) return;
mutex_lock(&evl->lock); status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); t = status.tail; h = status.head; size = evl->size;
while (h != t) { entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) set_bit(h, evl->bmap); h = (h + 1) % size; } if (wq->wq) drain_workqueue(wq->wq);
mutex_unlock(&evl->lock);}
static int idxd_cdev_release(struct inode *node, struct file *filep){ struct idxd_user_context *ctx = filep->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev;
dev_dbg(dev, "%s called\n", __func__); filep->private_data = NULL;
device_unregister(user_ctx_dev(ctx));
return 0;}
static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, const char *func){ struct device *dev = &wq->idxd->pdev->dev;
if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { dev_info_ratelimited(dev, "%s: %s: mapping too large: %lu\n", current->comm, func, vma->vm_end - vma->vm_start); return -EINVAL; }
return 0;}
static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma){ struct idxd_user_context *ctx = filp->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; struct pci_dev *pdev = idxd->pdev; phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); unsigned long pfn; int rc;
dev_dbg(&pdev->dev, "%s called\n", __func__);
/* * Due to an erratum in some of the devices supported by the driver, * direct user submission to the device can be unsafe. * (See the INTEL-SA-01084 security advisory) * * For the devices that exhibit this behavior, require that the user * has CAP_SYS_RAWIO capabilities. */ if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) return -EPERM;
if (current->mm != ctx->mm) return -EPERM;
rc = check_vma(wq, vma, __func__); if (rc < 0) return rc;
vm_flags_set(vma, VM_DONTCOPY); pfn = (base + idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vma->vm_private_data = ctx;
return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, vma->vm_page_prot);}
static int idxd_submit_user_descriptor(struct idxd_user_context *ctx, struct dsa_hw_desc __user *udesc){ struct idxd_wq *wq = ctx->wq; struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev; const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40; void __iomem *portal = idxd_wq_portal_addr(wq); struct dsa_hw_desc descriptor __aligned(64); int rc;
rc = copy_from_user(&descriptor, udesc, sizeof(descriptor)); if (rc) return -EFAULT;
/* * DSA devices are capable of indirect ("batch") command submission. * On devices where direct user submissions are not safe, we cannot * allow this since there is no good way for us to verify these * indirect commands. Narrow the restriction of operations with the * BATCH opcode to only DSA version 1 devices. */ if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH && wq->idxd->hw.version == DEVICE_VERSION_1 && !wq->idxd->user_submission_safe) return -EINVAL; /* * As per the programming specification, the completion address must be * aligned to 32 or 64 bytes. If this is violated the hardware * engine can get very confused (security issue). */ if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align)) return -EINVAL;
if (wq_dedicated(wq)) iosubmit_cmds512(portal, &descriptor, 1); else { descriptor.priv = 0; descriptor.pasid = ctx->pasid; rc = idxd_enqcmds(wq, portal, &descriptor); if (rc < 0) return rc; }
return 0;}
static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len, loff_t *unused){ struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf; struct idxd_user_context *ctx = filp->private_data; ssize_t written = 0; int i;
if (current->mm != ctx->mm) return -EPERM;
for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) { int rc = idxd_submit_user_descriptor(ctx, udesc + i);
if (rc) return written ? written : rc;
written += sizeof(struct dsa_hw_desc); }
return written;}
static __poll_t idxd_cdev_poll(struct file *filp, struct poll_table_struct *wait){ struct idxd_user_context *ctx = filp->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; __poll_t out = 0;
if (current->mm != ctx->mm) return POLLNVAL;
poll_wait(filp, &wq->err_queue, wait); spin_lock(&idxd->dev_lock); if (idxd->sw_err.valid) out = EPOLLIN | EPOLLRDNORM; spin_unlock(&idxd->dev_lock);
return out;}
static const struct file_operations idxd_cdev_fops = { .owner = THIS_MODULE, .open = idxd_cdev_open, .release = idxd_cdev_release, .mmap = idxd_cdev_mmap, .write = idxd_cdev_write, .poll = idxd_cdev_poll,};
int idxd_cdev_get_major(struct idxd_device *idxd){ return MAJOR(ictx[idxd->data->type].devt);}
int idxd_wq_add_cdev(struct idxd_wq *wq){ struct idxd_device *idxd = wq->idxd; struct idxd_cdev *idxd_cdev; struct cdev *cdev; struct device *dev; struct idxd_cdev_context *cdev_ctx; int rc, minor;
idxd_cdev = kzalloc_obj(*idxd_cdev); if (!idxd_cdev) return -ENOMEM;
idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; idxd_cdev->wq = wq; cdev = &idxd_cdev->cdev; dev = cdev_dev(idxd_cdev); cdev_ctx = &ictx[wq->idxd->data->type]; minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL); if (minor < 0) { kfree(idxd_cdev); return minor; } idxd_cdev->minor = minor;
device_initialize(dev); dev->parent = wq_confdev(wq); dev->bus = &dsa_bus_type; dev->type = &idxd_cdev_device_type; dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); if (rc < 0) goto err;
wq->idxd_cdev = idxd_cdev; cdev_init(cdev, &idxd_cdev_fops); rc = cdev_device_add(cdev, dev); if (rc) { dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); goto err; }
return 0;
err: put_device(dev); wq->idxd_cdev = NULL; return rc;}
void idxd_wq_del_cdev(struct idxd_wq *wq){ struct idxd_cdev_context *cdev_ctx; struct idxd_cdev *idxd_cdev;
idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
cdev_ctx = &ictx[wq->idxd->data->type]; ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); put_device(cdev_dev(idxd_cdev));}
static int idxd_user_drv_probe(struct idxd_dev *idxd_dev){ struct device *dev = &idxd_dev->conf_dev; struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); struct idxd_device *idxd = wq->idxd; int rc;
if (idxd->state != IDXD_DEV_ENABLED) return -ENXIO;
mutex_lock(&wq->wq_lock);
if (!idxd_wq_driver_name_match(wq, dev)) { idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; rc = -ENODEV; goto wq_err; }
/* * User type WQ is enabled only when SVA is enabled for two reasons: * - If no IOMMU or IOMMU Passthrough without SVA, userspace * can directly access physical address through the WQ. * - The IDXD cdev driver does not provide any ways to pin * user pages and translate the address from user VA to IOVA or * PA without IOMMU SVA. Therefore the application has no way * to instruct the device to perform DMA function. This makes * the cdev not usable for normal application usage. */ if (!device_user_pasid_enabled(idxd)) { idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; dev_dbg(&idxd->pdev->dev, "User type WQ cannot be enabled without SVA.\n");
rc = -EOPNOTSUPP; goto wq_err; }
wq->wq = create_workqueue(dev_name(wq_confdev(wq))); if (!wq->wq) { rc = -ENOMEM; goto wq_err; }
wq->type = IDXD_WQT_USER; rc = idxd_drv_enable_wq(wq); if (rc < 0) goto err;
rc = idxd_wq_add_cdev(wq); if (rc < 0) { idxd->cmd_status = IDXD_SCMD_CDEV_ERR; goto err_cdev; }
idxd->cmd_status = 0; mutex_unlock(&wq->wq_lock); return 0;
err_cdev: idxd_drv_disable_wq(wq);err: destroy_workqueue(wq->wq); wq->type = IDXD_WQT_NONE;wq_err: mutex_unlock(&wq->wq_lock); return rc;}
static void idxd_user_drv_remove(struct idxd_dev *idxd_dev){ struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
mutex_lock(&wq->wq_lock); idxd_wq_del_cdev(wq); idxd_drv_disable_wq(wq); wq->type = IDXD_WQT_NONE; destroy_workqueue(wq->wq); wq->wq = NULL; mutex_unlock(&wq->wq_lock);}
static enum idxd_dev_type dev_types[] = { IDXD_DEV_WQ, IDXD_DEV_NONE,};
struct idxd_device_driver idxd_user_drv = { .probe = idxd_user_drv_probe, .remove = idxd_user_drv_remove, .name = "user", .type = dev_types,};EXPORT_SYMBOL_GPL(idxd_user_drv);
int idxd_cdev_register(void){ int rc, i;
for (i = 0; i < IDXD_TYPE_MAX; i++) { ida_init(&ictx[i].minor_ida); rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, ictx[i].name); if (rc) goto err_free_chrdev_region; }
return 0;
err_free_chrdev_region: for (i--; i >= 0; i--) unregister_chrdev_region(ictx[i].devt, MINORMASK);
return rc;}
void idxd_cdev_remove(void){ int i;
for (i = 0; i < IDXD_TYPE_MAX; i++) { unregister_chrdev_region(ictx[i].devt, MINORMASK); ida_destroy(&ictx[i].minor_ida); }}
/** * idxd_copy_cr - copy completion record to user address space found by wq and * PASID * @wq: work queue * @pasid: PASID * @addr: user fault address to write * @cr: completion record * @len: number of bytes to copy * * This is called by a work that handles completion record fault. * * Return: number of bytes copied. */int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, void *cr, int len){ struct device *dev = &wq->idxd->pdev->dev; int left = len, status_size = 1; struct idxd_user_context *ctx; struct mm_struct *mm;
mutex_lock(&wq->uc_lock);
ctx = xa_load(&wq->upasid_xa, pasid); if (!ctx) { dev_warn(dev, "No user context\n"); goto out; }
mm = ctx->mm; /* * The completion record fault handling work is running in kernel * thread context. It temporarily switches to the mm to copy cr * to addr in the mm. */ kthread_use_mm(mm); left = copy_to_user((void __user *)addr + status_size, cr + status_size, len - status_size); /* * Copy status only after the rest of completion record is copied * successfully so that the user gets the complete completion record * when a non-zero status is polled. */ if (!left) { u8 status;
/* * Ensure that the completion record's status field is written * after the rest of the completion record has been written. * This ensures that the user receives the correct completion * record information once polling for a non-zero status. */ wmb(); status = *(u8 *)cr; if (put_user(status, (u8 __user *)addr)) left += status_size; } else { left += status_size; } kthread_unuse_mm(mm);
out: mutex_unlock(&wq->uc_lock);
return len - left;}