Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575// SPDX-License-Identifier: GPL-2.0-or-later/* * Devmem TCP * * Authors: Mina Almasry <almasrymina@google.com> * Willem de Bruijn <willemdebruijn.kernel@gmail.com> * Kaiyuan Zhang <kaiyuanz@google.com */
#include <linux/dma-buf.h>#include <linux/dma-resv.h>#include <linux/mm.h>#include <linux/netdevice.h>#include <linux/types.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/helpers.h>#include <net/page_pool/memory_provider.h>#include <net/sock.h>#include <net/tcp.h>
#include "devmem.h"#include "mp_dmabuf_devmem.h"
/* Device memory support */
static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
static const struct memory_provider_ops dmabuf_devmem_ops;
static struct net_iov *net_iov_free_pop(struct net_iov_area *area, u32 *freelist, u32 *free_count){ u32 idx;
if (unlikely(!*free_count)) return NULL;
idx = freelist[--(*free_count)]; return &area->niovs[idx];}
static void net_iov_free_push(struct net_iov_area *area, u32 *freelist, u32 *free_count, struct net_iov *niov){ if (WARN_ON_ONCE(net_iov_owner(niov) != area || *free_count >= area->num_niovs)) return;
freelist[(*free_count)++] = net_iov_idx(niov);}
static const struct dma_buf_attach_ops net_devmem_dmabuf_attach_ops = { .allow_peer2peer = true,};
static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref){ struct net_devmem_dmabuf_binding *binding = container_of(ref, struct net_devmem_dmabuf_binding, ref);
INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free); schedule_work(&binding->unbind_w);}
void __net_devmem_dmabuf_binding_free(struct work_struct *wq){ struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
if (binding->freelist) WARN(binding->free_count != binding->area.num_niovs, "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%u", binding->area.num_niovs, binding->free_count);
kvfree(binding->area.niovs); dma_resv_lock(binding->dmabuf->resv, NULL); dma_buf_unmap_attachment(binding->attachment, binding->sgt, binding->direction); dma_buf_unpin(binding->attachment); dma_resv_unlock(binding->dmabuf->resv); dma_buf_detach(binding->dmabuf, binding->attachment); dma_buf_put(binding->dmabuf); xa_destroy(&binding->bound_rxqs); percpu_ref_exit(&binding->ref); kvfree(binding->freelist); kvfree(binding->tx_vec); kfree(binding);}
static unsigned intnet_devmem_alloc_dmabuf_bulk(struct net_devmem_dmabuf_binding *binding, netmem_ref *netmems, unsigned int count){ struct net_iov *niov; unsigned int i;
spin_lock_bh(&binding->freelist_lock);
count = min(count, binding->free_count); for (i = 0; i < count; i++) { niov = net_iov_free_pop(&binding->area, binding->freelist, &binding->free_count); netmems[i] = net_iov_to_netmem(niov); }
spin_unlock_bh(&binding->freelist_lock);
return count;}
void net_devmem_free_dmabuf(struct net_iov *niov){ struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
spin_lock_bh(&binding->freelist_lock); net_iov_free_push(&binding->area, binding->freelist, &binding->free_count, niov); spin_unlock_bh(&binding->freelist_lock);}
void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding){ struct netdev_rx_queue *rxq; unsigned long xa_idx; unsigned int rxq_idx;
xa_erase(&net_devmem_dmabuf_bindings, binding->id);
/* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the * erase. */ synchronize_net();
if (binding->list.next) list_del(&binding->list);
xa_for_each(&binding->bound_rxqs, xa_idx, rxq) { const struct pp_memory_provider_params mp_params = { .mp_priv = binding, .mp_ops = &dmabuf_devmem_ops, };
rxq_idx = get_netdev_rx_queue_index(rxq);
netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params); }
percpu_ref_kill(&binding->ref);}
int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, struct net_devmem_dmabuf_binding *binding, struct netlink_ext_ack *extack){ struct pp_memory_provider_params mp_params = { .mp_priv = binding, .mp_ops = &dmabuf_devmem_ops, }; struct netdev_rx_queue *rxq; u32 xa_idx; int err;
if (binding->niov_shift != PAGE_SHIFT) mp_params.rx_page_size = 1U << binding->niov_shift;
err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack); if (err) return err;
rxq = __netif_get_rx_queue(dev, rxq_idx); err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b, GFP_KERNEL); if (err) goto err_close_rxq;
return 0;
err_close_rxq: netif_mp_close_rxq(dev, rxq_idx, &mp_params); return err;}
struct net_devmem_dmabuf_binding *net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, unsigned int niov_shift, struct netdev_nl_sock *priv, const struct sock *nl_sk, struct netlink_ext_ack *extack){ struct net_devmem_dmabuf_binding *binding; size_t niov_size = 1UL << niov_shift; static u32 id_alloc_next; struct scatterlist *sg; struct dma_buf *dmabuf; unsigned int sg_idx; size_t niov_idx; size_t i; long err;
if (!dma_dev) { NL_SET_ERR_MSG(extack, "Device doesn't support DMA"); return ERR_PTR(-EOPNOTSUPP); }
dmabuf = dma_buf_get(dmabuf_fd); if (IS_ERR(dmabuf)) return ERR_CAST(dmabuf);
binding = kzalloc_node(sizeof(*binding), GFP_KERNEL, dev_to_node(&dev->dev)); if (!binding) { err = -ENOMEM; goto err_put_dmabuf; }
binding->dev = dev; binding->vdev = vdev; write_pnet(&binding->net, sock_net(nl_sk)); binding->uid = sk_uid(nl_sk); binding->niov_shift = niov_shift; xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
err = percpu_ref_init(&binding->ref, net_devmem_dmabuf_binding_release, 0, GFP_KERNEL); if (err < 0) goto err_free_binding;
mutex_init(&binding->lock);
binding->dmabuf = dmabuf; binding->direction = direction;
binding->attachment = dma_buf_dynamic_attach(binding->dmabuf, dma_dev, &net_devmem_dmabuf_attach_ops, binding); if (IS_ERR(binding->attachment)) { err = PTR_ERR(binding->attachment); NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device"); goto err_exit_ref; }
dma_resv_lock(dmabuf->resv, NULL); err = dma_buf_pin(binding->attachment); if (err) { NL_SET_ERR_MSG(extack, "Failed to pin dmabuf"); goto err_unlock; }
binding->sgt = dma_buf_map_attachment(binding->attachment, direction); if (IS_ERR(binding->sgt)) { err = PTR_ERR(binding->sgt); NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment"); goto err_unpin; } dma_resv_unlock(dmabuf->resv);
err = dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_KERNEL, true, MAX_SCHEDULE_TIMEOUT); if (err < 0) { NL_SET_ERR_MSG(extack, "Failed to wait for dmabuf fences"); goto err_unmap; }
if (!IS_ALIGNED(dmabuf->size, niov_size)) { err = -EINVAL; NL_SET_ERR_MSG_FMT(extack, "dmabuf size %zu not aligned to niov size %zu", dmabuf->size, niov_size); goto err_unmap; }
if ((dmabuf->size >> niov_shift) > U32_MAX) { err = -E2BIG; NL_SET_ERR_MSG(extack, "dmabuf contains too many net_iovs"); goto err_unmap; }
binding->area.num_niovs = dmabuf->size >> niov_shift; if (direction == DMA_TO_DEVICE) { binding->tx_vec = kvmalloc_objs(struct net_iov *, binding->area.num_niovs); if (!binding->tx_vec) { err = -ENOMEM; goto err_unmap; } } else { spin_lock_init(&binding->freelist_lock); binding->freelist = kvmalloc_array(binding->area.num_niovs, sizeof(binding->freelist[0]), GFP_KERNEL); if (!binding->freelist) { err = -ENOMEM; goto err_unmap; } } binding->area.niovs = kvzalloc_objs(*binding->area.niovs, binding->area.num_niovs); if (!binding->area.niovs) { err = -ENOMEM; goto err_free_freelist; }
niov_idx = 0; for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) { dma_addr_t dma_addr = sg_dma_address(sg); size_t len = sg_dma_len(sg); struct net_iov *niov; size_t nr_niovs;
if (!IS_ALIGNED(dma_addr, niov_size) || !IS_ALIGNED(len, niov_size)) { err = -EINVAL; NL_SET_ERR_MSG_FMT(extack, "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu", &dma_addr, len, niov_size); goto err_free_niovs; }
nr_niovs = len >> niov_shift; for (i = 0; i < nr_niovs; i++, niov_idx++) { niov = &binding->area.niovs[niov_idx]; net_iov_init(niov, &binding->area, NET_IOV_DMABUF); if (net_mp_niov_set_dma_addr(niov, dma_addr)) { err = -EFAULT; NL_SET_ERR_MSG(extack, "dmabuf DMA address cannot be represented"); goto err_free_niovs; } if (direction == DMA_TO_DEVICE) binding->tx_vec[niov_idx] = niov; else net_iov_free_push(&binding->area, binding->freelist, &binding->free_count, niov); dma_addr += niov_size; } }
err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id, binding, xa_limit_32b, &id_alloc_next, GFP_KERNEL); if (err < 0) goto err_free_niovs;
list_add(&binding->list, &priv->bindings);
return binding;
err_free_niovs: kvfree(binding->area.niovs);err_free_freelist: kvfree(binding->freelist); kvfree(binding->tx_vec);err_unmap: dma_resv_lock(dmabuf->resv, NULL); dma_buf_unmap_attachment(binding->attachment, binding->sgt, direction);err_unpin: dma_buf_unpin(binding->attachment);err_unlock: dma_resv_unlock(dmabuf->resv); dma_buf_detach(dmabuf, binding->attachment);err_exit_ref: percpu_ref_exit(&binding->ref);err_free_binding: kfree(binding);err_put_dmabuf: dma_buf_put(dmabuf); return ERR_PTR(err);}
struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id){ struct net_devmem_dmabuf_binding *binding;
rcu_read_lock(); binding = xa_load(&net_devmem_dmabuf_bindings, id); if (binding) { if (!net_devmem_dmabuf_binding_get(binding)) binding = NULL; } rcu_read_unlock();
return binding;}
void net_devmem_net_iov_inc_ref(struct net_iov *niov){ net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));}
void net_devmem_net_iov_dec_ref(struct net_iov *niov){ net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));}
struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id){ struct net_devmem_dmabuf_binding *binding; struct net_device *dst_dev; struct dst_entry *dst; int err = 0;
binding = net_devmem_lookup_dmabuf(dmabuf_id); if (!binding || !binding->tx_vec || !net_eq(sock_net(sk), read_pnet(&binding->net)) || !uid_eq(sk_uid(sk), binding->uid)) { err = -EINVAL; goto out_err; }
rcu_read_lock(); dst = __sk_dst_get(sk); /* If dst is NULL (route expired), attempt to rebuild it. */ if (unlikely(!dst)) { if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) { err = -EHOSTUNREACH; goto out_unlock; } dst = __sk_dst_get(sk); if (unlikely(!dst)) { err = -ENODEV; goto out_unlock; } }
/* The dma-addrs in this binding are only reachable to the corresponding * net_device. */ dst_dev = dst_dev_rcu(dst); if (unlikely(!dst_dev) || unlikely(dst_dev != READ_ONCE(binding->dev) && dst_dev != READ_ONCE(binding->vdev))) { err = -ENODEV; goto out_unlock; }
rcu_read_unlock(); return binding;
out_unlock: rcu_read_unlock();out_err: if (binding) net_devmem_dmabuf_binding_put(binding);
return ERR_PTR(err);}
struct net_iov *net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t virt_addr, size_t *off, size_t *size){ if (virt_addr >= binding->dmabuf->size) return NULL;
*off = virt_addr % PAGE_SIZE; *size = PAGE_SIZE - *off;
return binding->tx_vec[virt_addr / PAGE_SIZE];}
/*** "Dmabuf devmem memory provider" ***/
int mp_dmabuf_devmem_init(struct page_pool *pool){ struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
if (!binding) return -EINVAL;
/* dma-buf dma addresses do not need and should not be used with * dma_sync_for_cpu/device. Force disable dma_sync. */ pool->dma_sync = false; pool->dma_sync_for_cpu = false;
if (pool->p.order != binding->niov_shift - PAGE_SHIFT) return -E2BIG;
net_devmem_dmabuf_binding_get(binding); return 0;}
netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp){ struct net_devmem_dmabuf_binding *binding = pool->mp_priv; netmem_ref *netmems = pool->alloc.cache; unsigned int allocated, i;
if (WARN_ON_ONCE(pool->alloc.count)) return 0;
allocated = net_devmem_alloc_dmabuf_bulk(binding, netmems, PP_ALLOC_CACHE_REFILL); if (unlikely(!allocated)) return 0;
for (i = 0; i < allocated; i++) net_mp_niov_set_page_pool(pool, netmem_to_net_iov(netmems[i]));
/* Return the last one, the rest stay in the page_pool cache. */ allocated--; pool->alloc.count = allocated; return netmems[allocated];}
void mp_dmabuf_devmem_destroy(struct page_pool *pool){ struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
net_devmem_dmabuf_binding_put(binding);}
bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem){ long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) return false;
if (WARN_ON_ONCE(refcount != 1)) return false;
net_mp_niov_clear_page_pool(netmem_to_net_iov(netmem)); net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
/* We don't want the page pool put_page()ing our net_iovs. */ return false;}
static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp, struct netdev_rx_queue *rxq){ const struct net_devmem_dmabuf_binding *binding = mp_priv; int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
return nla_put_u32(rsp, type, binding->id);}
static void mp_dmabuf_devmem_uninstall(void *mp_priv, struct netdev_rx_queue *rxq){ struct net_devmem_dmabuf_binding *binding = mp_priv; struct netdev_rx_queue *bound_rxq; unsigned long xa_idx;
xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) { if (bound_rxq == rxq) { xa_erase(&binding->bound_rxqs, xa_idx); if (xa_empty(&binding->bound_rxqs)) { mutex_lock(&binding->lock); ASSERT_EXCLUSIVE_WRITER(binding->dev); WRITE_ONCE(binding->dev, NULL); mutex_unlock(&binding->lock); } break; } }}
static const struct memory_provider_ops dmabuf_devmem_ops = { .init = mp_dmabuf_devmem_init, .destroy = mp_dmabuf_devmem_destroy, .alloc_netmems = mp_dmabuf_devmem_alloc_netmems, .release_netmem = mp_dmabuf_devmem_release_page, .nl_fill = mp_dmabuf_devmem_nl_fill, .uninstall = mp_dmabuf_devmem_uninstall,};