Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488// SPDX-License-Identifier: GPL-2.0
#include <linux/mutex.h>#include <linux/netdevice.h>#include <linux/xarray.h>#include <net/busy_poll.h>#include <net/net_debug.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/helpers.h>#include <net/page_pool/types.h>#include <net/page_pool/memory_provider.h>#include <net/sock.h>
#include "page_pool_priv.h"#include "netdev-genl-gen.h"
static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);/* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev, * pool->user. * Ordering: inside rtnl_lock */DEFINE_MUTEX(page_pools_lock);
/* Page pools are only reachable from user space (via netlink) if they are * linked to a netdev at creation time. Following page pool "visibility" * states are possible: * - normal * - user.list: linked to real netdev, netdev: real netdev * - orphaned - real netdev has disappeared * - user.list: linked to lo, netdev: lo * - invisible - either (a) created without netdev linking, (b) unlisted due * to error, or (c) the entire namespace which owned this pool disappeared * - user.list: unhashed, netdev: unknown */
typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, const struct genl_info *info);
static intnetdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill){ struct page_pool *pool; struct sk_buff *rsp; int err;
mutex_lock(&page_pools_lock); pool = xa_load(&page_pools, id); if (!pool || hlist_unhashed(&pool->user.list) || !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { err = -ENOENT; goto err_unlock; }
rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!rsp) { err = -ENOMEM; goto err_unlock; }
err = fill(rsp, pool, info); if (err) goto err_free_msg;
mutex_unlock(&page_pools_lock);
return genlmsg_reply(rsp, info);
err_free_msg: nlmsg_free(rsp);err_unlock: mutex_unlock(&page_pools_lock); return err;}
struct page_pool_dump_cb { unsigned long ifindex; u32 pp_id;};
static intnetdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, pp_nl_fill_cb fill, struct nlattr *ifindex_attr){ struct page_pool_dump_cb *state = (void *)cb->ctx; const struct genl_info *info = genl_info_dump(cb); struct net *net = sock_net(skb->sk); unsigned long start_ifindex; struct net_device *netdev; struct page_pool *pool; int err = 0;
if (ifindex_attr) state->ifindex = nla_get_u32(ifindex_attr);
start_ifindex = state->ifindex;
rtnl_lock(); mutex_lock(&page_pools_lock); for_each_netdev_dump(net, netdev, state->ifindex) { /* Either the provided ifindex doesn't exist or done dumping */ if (ifindex_attr && netdev->ifindex != nla_get_u32(ifindex_attr)) break;
if (state->ifindex != start_ifindex) state->pp_id = 0;
hlist_for_each_entry(pool, &netdev->page_pools, user.list) { if (state->pp_id && state->pp_id < pool->user.id) continue;
state->pp_id = pool->user.id; err = fill(skb, pool, info); if (err) goto out; } }out: mutex_unlock(&page_pools_lock); rtnl_unlock();
return err;}
static intpage_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, const struct genl_info *info){#ifdef CONFIG_PAGE_POOL_STATS struct page_pool_stats stats = {}; struct nlattr *nest; void *hdr;
page_pool_get_stats(pool, &stats);
hdr = genlmsg_iput(rsp, info); if (!hdr) return -EMSGSIZE;
nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); if (!nest) goto err_cancel_msg;
if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, pool->slow.netdev->ifindex))) goto err_cancel_nest;
nla_nest_end(rsp, nest);
if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST, stats.alloc_stats.fast) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, stats.alloc_stats.slow) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, stats.alloc_stats.slow_high_order) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, stats.alloc_stats.empty) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, stats.alloc_stats.refill) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, stats.alloc_stats.waive) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, stats.recycle_stats.cached) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, stats.recycle_stats.cache_full) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, stats.recycle_stats.ring) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, stats.recycle_stats.ring_full) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, stats.recycle_stats.released_refcnt)) goto err_cancel_msg;
genlmsg_end(rsp, hdr);
return 0;err_cancel_nest: nla_nest_cancel(rsp, nest);err_cancel_msg: genlmsg_cancel(rsp, hdr); return -EMSGSIZE;#else GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS"); return -EOPNOTSUPP;#endif}
int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, struct genl_info *info){ struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)]; struct nlattr *nest; int err; u32 id;
if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO)) return -EINVAL;
nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, netdev_page_pool_info_nl_policy, info->extack); if (err) return err;
if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID)) return -EINVAL; if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) { NL_SET_ERR_MSG_ATTR(info->extack, tb[NETDEV_A_PAGE_POOL_IFINDEX], "selecting by ifindex not supported"); return -EINVAL; }
id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]);
return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);}
static const struct netlink_range_validation page_pool_ifindex_range = { .min = 1ULL, .max = S32_MAX,};
static const struct nla_policypage_pool_stat_info_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = { [NETDEV_A_PAGE_POOL_IFINDEX] = NLA_POLICY_FULL_RANGE(NLA_U32, &page_pool_ifindex_range),};
int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb){ struct nlattr *tb[ARRAY_SIZE(page_pool_stat_info_policy)]; const struct genl_info *info = genl_info_dump(cb); struct nlattr *ifindex_attr = NULL;
if (info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]) { struct nlattr *nest; int err;
nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, page_pool_stat_info_policy, info->extack); if (err) return err;
ifindex_attr = tb[NETDEV_A_PAGE_POOL_IFINDEX]; }
return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill, ifindex_attr);}
static intpage_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, const struct genl_info *info){ size_t inflight, refsz; unsigned int napi_id; void *hdr;
hdr = genlmsg_iput(rsp, info); if (!hdr) return -EMSGSIZE;
if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) goto err_cancel;
if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, pool->slow.netdev->ifindex)) goto err_cancel;
napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0; if (napi_id_valid(napi_id) && nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, napi_id)) goto err_cancel;
inflight = page_pool_inflight(pool, false); refsz = PAGE_SIZE << pool->p.order; if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, inflight * refsz)) goto err_cancel; if (pool->user.detach_time && nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, ktime_divns(pool->user.detach_time, NSEC_PER_SEC))) goto err_cancel;
if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL)) goto err_cancel;
genlmsg_end(rsp, hdr);
return 0;err_cancel: genlmsg_cancel(rsp, hdr); return -EMSGSIZE;}
static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd){ struct genl_info info; struct sk_buff *ntf; struct net *net;
lockdep_assert_held(&page_pools_lock);
/* 'invisible' page pools don't matter */ if (hlist_unhashed(&pool->user.list)) return; net = dev_net(pool->slow.netdev);
if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) return;
genl_info_init_ntf(&info, &netdev_nl_family, cmd);
ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!ntf) return;
if (page_pool_nl_fill(ntf, pool, &info)) { nlmsg_free(ntf); return; }
genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);}
int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info){ u32 id;
if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) return -EINVAL;
id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]);
return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);}
int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb){ const struct genl_info *info = genl_info_dump(cb);
return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill, info->attrs[NETDEV_A_PAGE_POOL_IFINDEX]);}
int page_pool_list(struct page_pool *pool){ static u32 id_alloc_next; int err;
mutex_lock(&page_pools_lock); err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, &id_alloc_next, GFP_KERNEL); if (err < 0) goto err_unlock;
INIT_HLIST_NODE(&pool->user.list); if (pool->slow.netdev) { hlist_add_head(&pool->user.list, &pool->slow.netdev->page_pools); netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); }
mutex_unlock(&page_pools_lock); return 0;
err_unlock: mutex_unlock(&page_pools_lock); return err;}
void page_pool_detached(struct page_pool *pool){ mutex_lock(&page_pools_lock); pool->user.detach_time = ktime_get_boottime(); netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); mutex_unlock(&page_pools_lock);}
void page_pool_unlist(struct page_pool *pool){ mutex_lock(&page_pools_lock); netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); xa_erase(&page_pools, pool->user.id); if (!hlist_unhashed(&pool->user.list)) hlist_del(&pool->user.list); mutex_unlock(&page_pools_lock);}
int page_pool_check_memory_provider(struct net_device *dev, struct netdev_rx_queue *rxq){ void *binding = rxq->mp_params.mp_priv; struct page_pool *pool; struct hlist_node *n;
if (!binding) return 0;
mutex_lock(&page_pools_lock); hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) { if (pool->mp_priv != binding) continue;
if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) { mutex_unlock(&page_pools_lock); return 0; } } mutex_unlock(&page_pools_lock); return -ENODATA;}
static void page_pool_unreg_netdev_wipe(struct net_device *netdev){ struct page_pool *pool; struct hlist_node *n;
mutex_lock(&page_pools_lock); hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { hlist_del_init(&pool->user.list); pool->slow.netdev = NET_PTR_POISON; } mutex_unlock(&page_pools_lock);}
static void page_pool_unreg_netdev(struct net_device *netdev){ struct page_pool *pool, *last; struct net_device *lo;
lo = dev_net(netdev)->loopback_dev;
mutex_lock(&page_pools_lock); last = NULL; hlist_for_each_entry(pool, &netdev->page_pools, user.list) { pool->slow.netdev = lo; netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); last = pool; } if (last) hlist_splice_init(&netdev->page_pools, &last->user.list, &lo->page_pools); mutex_unlock(&page_pools_lock);}
static intpage_pool_netdevice_event(struct notifier_block *nb, unsigned long event, void *ptr){ struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
if (event != NETDEV_UNREGISTER) return NOTIFY_DONE;
if (hlist_empty(&netdev->page_pools)) return NOTIFY_OK;
if (netdev->ifindex != LOOPBACK_IFINDEX) page_pool_unreg_netdev(netdev); else page_pool_unreg_netdev_wipe(netdev); return NOTIFY_OK;}
static struct notifier_block page_pool_netdevice_nb = { .notifier_call = page_pool_netdevice_event,};
static int __init page_pool_user_init(void){ return register_netdevice_notifier(&page_pool_netdevice_nb);}
subsys_initcall(page_pool_user_init);