Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299/* SPDX-License-Identifier: GPL-2.0-only *//* * Copyright (C) 2012 Red Hat, Inc. All rights reserved. * Author: Alex Williamson <alex.williamson@redhat.com> * * Derived from original vfio: * Copyright 2010 Cisco Systems, Inc. All rights reserved. * Author: Tom Lyon, pugs@cisco.com */
#include <linux/device.h>#include <linux/mutex.h>#include <linux/pci.h>#include <linux/vfio.h>#include <linux/irqbypass.h>#include <linux/rcupdate.h>#include <linux/types.h>#include <linux/uuid.h>#include <linux/notifier.h>#include <linux/pm.h>
#ifndef VFIO_PCI_CORE_H#define VFIO_PCI_CORE_H
#define VFIO_PCI_OFFSET_SHIFT 40#define VFIO_PCI_OFFSET_TO_INDEX(off) ((u64)(off) >> VFIO_PCI_OFFSET_SHIFT)#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
struct vfio_pci_core_device;struct vfio_pci_region;struct p2pdma_provider;struct dma_buf_attachment;
struct vfio_pci_eventfd { struct eventfd_ctx *ctx; struct rcu_head rcu;};
struct vfio_pci_regops { ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite); void (*release)(struct vfio_pci_core_device *vdev, struct vfio_pci_region *region); int (*mmap)(struct vfio_pci_core_device *vdev, struct vfio_pci_region *region, struct vm_area_struct *vma); int (*add_capability)(struct vfio_pci_core_device *vdev, struct vfio_pci_region *region, struct vfio_info_cap *caps);};
struct vfio_pci_region { u32 type; u32 subtype; const struct vfio_pci_regops *ops; void *data; size_t size; u32 flags;};
struct vfio_pci_device_ops { int (*get_dmabuf_phys)(struct vfio_pci_core_device *vdev, struct p2pdma_provider **provider, unsigned int region_index, struct phys_vec *phys_vec, struct vfio_region_dma_range *dma_ranges, size_t nr_ranges);};
#if IS_ENABLED(CONFIG_VFIO_PCI_DMABUF)int vfio_pci_core_fill_phys_vec(struct phys_vec *phys_vec, struct vfio_region_dma_range *dma_ranges, size_t nr_ranges, phys_addr_t start, phys_addr_t len);int vfio_pci_core_get_dmabuf_phys(struct vfio_pci_core_device *vdev, struct p2pdma_provider **provider, unsigned int region_index, struct phys_vec *phys_vec, struct vfio_region_dma_range *dma_ranges, size_t nr_ranges);#elsestatic inline intvfio_pci_core_fill_phys_vec(struct phys_vec *phys_vec, struct vfio_region_dma_range *dma_ranges, size_t nr_ranges, phys_addr_t start, phys_addr_t len){ return -EINVAL;}static inline int vfio_pci_core_get_dmabuf_phys( struct vfio_pci_core_device *vdev, struct p2pdma_provider **provider, unsigned int region_index, struct phys_vec *phys_vec, struct vfio_region_dma_range *dma_ranges, size_t nr_ranges){ return -EOPNOTSUPP;}#endif
struct vfio_pci_core_device { struct vfio_device vdev; struct pci_dev *pdev; const struct vfio_pci_device_ops *pci_ops; void __iomem *barmap[PCI_STD_NUM_BARS]; bool bar_mmap_supported[PCI_STD_NUM_BARS]; /* Flags modified at runtime - dedicated storage unit */ bool virq_disabled; bool bardirty; u8 *pci_config_map; u8 *vconfig; struct perm_bits *msi_perm; spinlock_t irqlock; struct mutex igate; struct xarray ctx; int irq_type; int num_regions; struct vfio_pci_region *region; u8 msi_qmax; u8 msix_bar; u16 msix_size; u32 msix_offset; u32 rbar[7]; /* Flags only modified on setup/release - bitfield ok */ bool has_dyn_msix:1; bool pci_2_3:1; bool reset_works:1; bool extended_caps:1; bool has_vga:1; bool nointx:1; bool needs_pm_restore:1; bool disable_idle_d3:1; bool nointxmask:1; bool disable_vga:1; /* Flags modified at runtime - dedicated storage unit */ bool needs_reset; bool pm_intx_masked; bool pm_runtime_engaged; bool sriov_active; struct pci_saved_state *pci_saved_state; struct pci_saved_state *pm_save; int ioeventfds_nr; struct vfio_pci_eventfd __rcu *err_trigger; struct vfio_pci_eventfd __rcu *req_trigger; struct eventfd_ctx *pm_wake_eventfd_ctx; struct list_head dummy_resources_list; struct mutex ioeventfds_lock; struct list_head ioeventfds_list; struct vfio_pci_vf_token *vf_token; struct list_head sriov_pfs_item; struct vfio_pci_core_device *sriov_pf_core_dev; struct notifier_block nb; struct rw_semaphore memory_lock; struct list_head dmabufs;};
enum vfio_pci_io_width { VFIO_PCI_IO_WIDTH_1 = 1, VFIO_PCI_IO_WIDTH_2 = 2, VFIO_PCI_IO_WIDTH_4 = 4, VFIO_PCI_IO_WIDTH_8 = 8,};
/* Will be exported for vfio pci drivers usage */int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, unsigned int type, unsigned int subtype, const struct vfio_pci_regops *ops, size_t size, u32 flags, void *data);void vfio_pci_core_close_device(struct vfio_device *core_vdev);int vfio_pci_core_init_dev(struct vfio_device *core_vdev);void vfio_pci_core_release_dev(struct vfio_device *core_vdev);int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev);void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev);int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, int nr_virtfn);long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, unsigned long arg);int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, void __user *arg, size_t argsz);int vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev, struct vfio_region_info *info, struct vfio_info_cap *caps);ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, size_t count, loff_t *ppos);ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, size_t count, loff_t *ppos);vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, struct vm_fault *vmf, unsigned long pfn, unsigned int order);int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma);void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev, const uuid_t *uuid);int vfio_pci_core_enable(struct vfio_pci_core_device *vdev);void vfio_pci_core_disable(struct vfio_pci_core_device *vdev);void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev);ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, void __iomem *io, char __user *buf, loff_t off, size_t count, size_t x_start, size_t x_end, bool iswrite, enum vfio_pci_io_width max_width);bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev);bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, loff_t reg_start, size_t reg_cnt, loff_t *buf_offset, size_t *intersect_count, size_t *register_offset);#define VFIO_IOWRITE_DECLARATION(size) \int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io);
VFIO_IOWRITE_DECLARATION(8)VFIO_IOWRITE_DECLARATION(16)VFIO_IOWRITE_DECLARATION(32)#ifdef iowrite64VFIO_IOWRITE_DECLARATION(64)#endif
#define VFIO_IOREAD_DECLARATION(size) \int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size *val, void __iomem *io);
VFIO_IOREAD_DECLARATION(8)VFIO_IOREAD_DECLARATION(16)VFIO_IOREAD_DECLARATION(32)#ifdef ioread64VFIO_IOREAD_DECLARATION(64)#endif
static inline bool is_aligned_for_order(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned int order){ return !(order && (addr < vma->vm_start || addr + (PAGE_SIZE << order) > vma->vm_end || !IS_ALIGNED(pfn, 1 << order)));}
/* * Returns a BAR's iomap base or an ERR_PTR() if, for example, the * BAR isn't valid, its resource wasn't acquired, or its iomap * failed. This shall only be used after vfio_pci_core_enable() * has set up the BAR maps and before vfio_pci_core_disable() * tears them down. */static inline void __iomem __must_check *vfio_pci_core_get_iomap(struct vfio_pci_core_device *vdev, unsigned int bar){ if (WARN_ON_ONCE(bar >= PCI_STD_NUM_BARS)) return IOMEM_ERR_PTR(-EINVAL);
if (WARN_ON_ONCE(!vdev->barmap[bar])) return IOMEM_ERR_PTR(-ENODEV);
return vdev->barmap[bar];}
int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, struct phys_vec *phys);
int vfio_pci_core_runtime_suspend(struct vfio_pci_core_device *vdev);int vfio_pci_core_runtime_resume(struct vfio_pci_core_device *vdev);pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev);
/* * Per-driver PM/AER trampolines. driver_data points to the driver's private * structure, which embeds a struct vfio_pci_core_device. * * @name: prefix for <name>_pm_ops and <name>_aer_err_detected * @drv_type: type of the driver's private structure, including struct * @member: member containing the struct vfio_pci_core_device */#define VFIO_PCI_CORE_DEFINE_CALLBACKS(name, drv_type, member) \static int __maybe_unused name##_pm_suspend(struct device *dev) \{ \ drv_type *drvdata = dev_get_drvdata(dev); \ \ return vfio_pci_core_runtime_suspend(&drvdata->member); \} \static int __maybe_unused name##_pm_resume(struct device *dev) \{ \ drv_type *drvdata = dev_get_drvdata(dev); \ \ return vfio_pci_core_runtime_resume(&drvdata->member); \} \static const struct dev_pm_ops name##_pm_ops = { \ SET_RUNTIME_PM_OPS(name##_pm_suspend, name##_pm_resume, NULL) \}; \static pci_ers_result_t name##_aer_err_detected(struct pci_dev *pdev, \ pci_channel_state_t state) \{ \ drv_type *drvdata = pci_get_drvdata(pdev); \ \ return vfio_pci_core_aer_err_detected(&drvdata->member); \}
#endif /* VFIO_PCI_CORE_H */