Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287/* * Copyright (C) 2017 Netronome Systems, Inc. * * This software is licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this * source tree. * * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. */
#include <linux/debugfs.h>#include <linux/etherdevice.h>#include <linux/ethtool_netlink.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/netdevice.h>#include <linux/slab.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/helpers.h>#include <net/netlink.h>#include <net/net_shaper.h>#include <net/netdev_lock.h>#include <net/pkt_cls.h>#include <net/rtnetlink.h>#include <net/udp_tunnel.h>#include <net/busy_poll.h>
#include "netdevsim.h"
MODULE_IMPORT_NS("NETDEV_INTERNAL");
#define NSIM_RING_SIZE 256
static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq){ struct netdevsim *ns = netdev_priv(dev); struct net_device *peer_dev; struct netdevsim *peer_ns; struct netdev_queue *txq; u16 idx;
idx = rq->napi.index; rcu_read_lock(); peer_ns = rcu_dereference(ns->peer); if (!peer_ns) goto out;
/* TX device */ peer_dev = peer_ns->netdev; if (dev->real_num_tx_queues != peer_dev->num_rx_queues) goto out;
txq = netdev_get_tx_queue(peer_dev, idx); if (!netif_tx_queue_stopped(txq)) goto out;
netif_tx_wake_queue(txq);out: rcu_read_unlock();}
static void nsim_stop_tx_queue(struct net_device *tx_dev, struct net_device *rx_dev, struct nsim_rq *rq, u16 idx){ /* If different queues size, do not stop, since it is not * easy to find which TX queue is mapped here */ if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) return;
/* rq is the queue on the receive side */ netif_subqueue_try_stop(tx_dev, idx, NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), NSIM_RING_SIZE / 2);}
static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, struct nsim_rq *rq, struct sk_buff *skb){ if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { dev_kfree_skb_any(skb); return NET_RX_DROP; }
skb_queue_tail(&rq->skb_queue, skb);
/* Stop the peer TX queue avoiding dropping packets later */ if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) nsim_stop_tx_queue(tx_dev, rx_dev, rq, skb_get_queue_mapping(skb));
return NET_RX_SUCCESS;}
static int nsim_forward_skb(struct net_device *tx_dev, struct net_device *rx_dev, struct sk_buff *skb, struct nsim_rq *rq, struct skb_ext *psp_ext){ int ret;
ret = __dev_forward_skb(rx_dev, skb); if (ret) { if (psp_ext) __skb_ext_put(psp_ext); return ret; }
nsim_psp_handle_ext(skb, psp_ext);
return nsim_napi_rx(tx_dev, rx_dev, rq, skb);}
static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev){ struct netdevsim *ns = netdev_priv(dev); struct skb_ext *psp_ext = NULL; struct net_device *peer_dev; unsigned int len = skb->len; struct netdevsim *peer_ns; struct netdev_config *cfg; struct nsim_rq *rq; int rxq; int dr;
rcu_read_lock(); if (!nsim_ipsec_tx(ns, skb)) goto out_drop_any;
/* Check if loopback mode is enabled */ if (dev->features & NETIF_F_LOOPBACK) { peer_ns = ns; peer_dev = dev; } else { peer_ns = rcu_dereference(ns->peer); if (!peer_ns) goto out_drop_any; peer_dev = peer_ns->netdev; }
dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext); if (dr) goto out_drop_free;
rxq = skb_get_queue_mapping(skb); if (rxq >= peer_dev->num_rx_queues) rxq = rxq % peer_dev->num_rx_queues; rq = peer_ns->rq[rxq];
cfg = peer_dev->cfg; if (skb_is_nonlinear(skb) && (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && cfg->hds_thresh > len))) skb_linearize(skb);
skb_tx_timestamp(skb); if (unlikely(nsim_forward_skb(dev, peer_dev, skb, rq, psp_ext) == NET_RX_DROP)) goto out_drop_cnt;
if (!hrtimer_active(&rq->napi_timer)) hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL);
rcu_read_unlock(); dev_dstats_tx_add(dev, len); return NETDEV_TX_OK;
out_drop_any: dr = SKB_DROP_REASON_NOT_SPECIFIED;out_drop_free: kfree_skb_reason(skb, dr);out_drop_cnt: rcu_read_unlock(); dev_dstats_tx_dropped(dev); return NETDEV_TX_OK;}
static netdev_tx_t nsim_start_xmit_vf(struct sk_buff *skb, struct net_device *dev){ dev_dstats_tx_dropped(dev); kfree_skb(skb); return NETDEV_TX_OK;}
static int nsim_set_rx_mode(struct net_device *dev, struct netdev_hw_addr_list *uc, struct netdev_hw_addr_list *mc){ return 0;}
static int nsim_change_mtu(struct net_device *dev, int new_mtu){ struct netdevsim *ns = netdev_priv(dev);
if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && new_mtu > NSIM_XDP_MAX_MTU) return -EBUSY;
WRITE_ONCE(dev->mtu, new_mtu);
return 0;}
static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
/* Only refuse multicast addresses, zero address can mean unset/any. */ if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) return -EINVAL; memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
return 0;}
static int nsim_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, __be16 vlan_proto){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) return -EINVAL;
nsim_dev->vfconfigs[vf].vlan = vlan; nsim_dev->vfconfigs[vf].qos = qos; nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto;
return 0;}
static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { pr_err("Not supported in switchdev mode. Please use devlink API.\n"); return -EOPNOTSUPP; }
if (vf >= nsim_dev_get_vfs(nsim_dev)) return -EINVAL;
nsim_dev->vfconfigs[vf].min_tx_rate = min; nsim_dev->vfconfigs[vf].max_tx_rate = max;
return 0;}
static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (vf >= nsim_dev_get_vfs(nsim_dev)) return -EINVAL; nsim_dev->vfconfigs[vf].spoofchk_enabled = val;
return 0;}
static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (vf >= nsim_dev_get_vfs(nsim_dev)) return -EINVAL; nsim_dev->vfconfigs[vf].rss_query_enabled = val;
return 0;}
static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (vf >= nsim_dev_get_vfs(nsim_dev)) return -EINVAL; nsim_dev->vfconfigs[vf].trusted = val;
return 0;}
static intnsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (vf >= nsim_dev_get_vfs(nsim_dev)) return -EINVAL;
ivi->vf = vf; ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; ivi->vlan = nsim_dev->vfconfigs[vf].vlan; ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; ivi->qos = nsim_dev->vfconfigs[vf].qos; memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; ivi->trusted = nsim_dev->vfconfigs[vf].trusted; ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled;
return 0;}
static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state){ struct netdevsim *ns = netdev_priv(dev); struct nsim_dev *nsim_dev = ns->nsim_dev;
if (vf >= nsim_dev_get_vfs(nsim_dev)) return -EINVAL;
switch (state) { case IFLA_VF_LINK_STATE_AUTO: case IFLA_VF_LINK_STATE_ENABLE: case IFLA_VF_LINK_STATE_DISABLE: break; default: return -EINVAL; }
nsim_dev->vfconfigs[vf].link_state = state;
return 0;}
static intnsim_set_features(struct net_device *dev, netdev_features_t features){ struct netdevsim *ns = netdev_priv(dev);
if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) return nsim_bpf_disable_tc(ns);
return 0;}
static int nsim_get_iflink(const struct net_device *dev){ struct netdevsim *nsim, *peer; int iflink;
nsim = netdev_priv(dev);
rcu_read_lock(); peer = rcu_dereference(nsim->peer); iflink = peer ? READ_ONCE(peer->netdev->ifindex) : READ_ONCE(dev->ifindex); rcu_read_unlock();
return iflink;}
static int nsim_rcv(struct nsim_rq *rq, int budget){ struct net_device *dev = rq->napi.dev; struct bpf_prog *xdp_prog; struct netdevsim *ns; struct sk_buff *skb; unsigned int skblen; int i, ret;
ns = netdev_priv(dev); xdp_prog = READ_ONCE(ns->xdp.prog);
for (i = 0; i < budget; i++) { if (skb_queue_empty(&rq->skb_queue)) break;
skb = skb_dequeue(&rq->skb_queue);
if (xdp_prog) { /* skb might be freed directly by XDP, save the len */ skblen = skb->len;
if (skb->ip_summed == CHECKSUM_PARTIAL) skb_checksum_help(skb); ret = do_xdp_generic(xdp_prog, &skb); if (ret != XDP_PASS) { dev_dstats_rx_add(dev, skblen); continue; } }
/* skb might be discard at netif_receive_skb, save the len */ dev_dstats_rx_add(dev, skb->len); napi_gro_receive(&rq->napi, skb); }
nsim_start_peer_tx_queue(dev, rq); return i;}
static int nsim_poll(struct napi_struct *napi, int budget){ struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); int done;
done = nsim_rcv(rq, budget); if (done < budget) napi_complete_done(napi, done);
return done;}
static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi){ struct page_pool_params params = { .order = 0, .pool_size = NSIM_RING_SIZE, .nid = NUMA_NO_NODE, .dev = &napi->dev->dev, .napi = napi, .dma_dir = DMA_BIDIRECTIONAL, .netdev = napi->dev, }; struct page_pool *pool;
pool = page_pool_create(¶ms); if (IS_ERR(pool)) return PTR_ERR(pool);
*p = pool; return 0;}
static int nsim_init_napi(struct netdevsim *ns){ struct net_device *dev = ns->netdev; struct nsim_rq *rq; int err, i;
for (i = 0; i < dev->num_rx_queues; i++) { rq = ns->rq[i];
netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); }
for (i = 0; i < dev->num_rx_queues; i++) { rq = ns->rq[i];
err = nsim_create_page_pool(&rq->page_pool, &rq->napi); if (err) goto err_pp_destroy; }
return 0;
err_pp_destroy: while (i--) { page_pool_destroy(ns->rq[i]->page_pool); ns->rq[i]->page_pool = NULL; }
for (i = 0; i < dev->num_rx_queues; i++) __netif_napi_del_locked(&ns->rq[i]->napi);
return err;}
static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer){ struct nsim_rq *rq;
rq = container_of(timer, struct nsim_rq, napi_timer); napi_schedule(&rq->napi);
return HRTIMER_NORESTART;}
static void nsim_rq_timer_init(struct nsim_rq *rq){ hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, HRTIMER_MODE_REL);}
static void nsim_enable_napi(struct netdevsim *ns){ struct net_device *dev = ns->netdev; int i;
for (i = 0; i < dev->num_rx_queues; i++) { struct nsim_rq *rq = ns->rq[i];
netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); napi_enable_locked(&rq->napi); }}
static int nsim_open(struct net_device *dev){ struct netdevsim *ns = netdev_priv(dev); struct netdevsim *peer; int err;
netdev_assert_locked(dev);
err = nsim_init_napi(ns); if (err) return err;
nsim_enable_napi(ns);
peer = rtnl_dereference(ns->peer); if (peer && netif_running(peer->netdev)) { netif_carrier_on(dev); netif_carrier_on(peer->netdev); }
return 0;}
static void nsim_del_napi(struct netdevsim *ns){ struct net_device *dev = ns->netdev; int i;
for (i = 0; i < dev->num_rx_queues; i++) { struct nsim_rq *rq = ns->rq[i];
netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL); napi_disable_locked(&rq->napi); __netif_napi_del_locked(&rq->napi); } synchronize_net();
for (i = 0; i < dev->num_rx_queues; i++) { page_pool_destroy(ns->rq[i]->page_pool); ns->rq[i]->page_pool = NULL; }}
static int nsim_stop(struct net_device *dev){ struct netdevsim *ns = netdev_priv(dev); struct netdevsim *peer;
netdev_assert_locked(dev);
netif_carrier_off(dev); peer = rtnl_dereference(ns->peer); if (peer) netif_carrier_off(peer->netdev);
nsim_del_napi(ns);
return 0;}
static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid){ struct netdevsim *ns = netdev_priv(dev);
if (vid >= VLAN_N_VID) return -EINVAL;
if (proto == htons(ETH_P_8021Q)) WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag)); else if (proto == htons(ETH_P_8021AD)) WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag));
return 0;}
static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid){ struct netdevsim *ns = netdev_priv(dev);
if (vid >= VLAN_N_VID) return -EINVAL;
if (proto == htons(ETH_P_8021Q)) WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag)); else if (proto == htons(ETH_P_8021AD)) WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag));
return 0;}
static int nsim_shaper_set(struct net_shaper_binding *binding, const struct net_shaper *shaper, struct netlink_ext_ack *extack){ return 0;}
static int nsim_shaper_del(struct net_shaper_binding *binding, const struct net_shaper_handle *handle, struct netlink_ext_ack *extack){ return 0;}
static int nsim_shaper_group(struct net_shaper_binding *binding, int leaves_count, const struct net_shaper *leaves, const struct net_shaper *root, struct netlink_ext_ack *extack){ return 0;}
static void nsim_shaper_cap(struct net_shaper_binding *binding, enum net_shaper_scope scope, unsigned long *flags){ *flags = ULONG_MAX;}
static const struct net_shaper_ops nsim_shaper_ops = { .set = nsim_shaper_set, .delete = nsim_shaper_del, .group = nsim_shaper_group, .capabilities = nsim_shaper_cap,};
static const struct net_device_ops nsim_netdev_ops = { .ndo_start_xmit = nsim_start_xmit, .ndo_set_rx_mode_async = nsim_set_rx_mode, .ndo_set_mac_address = eth_mac_addr, .ndo_validate_addr = eth_validate_addr, .ndo_change_mtu = nsim_change_mtu, .ndo_set_vf_mac = nsim_set_vf_mac, .ndo_set_vf_vlan = nsim_set_vf_vlan, .ndo_set_vf_rate = nsim_set_vf_rate, .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, .ndo_set_vf_trust = nsim_set_vf_trust, .ndo_get_vf_config = nsim_get_vf_config, .ndo_set_vf_link_state = nsim_set_vf_link_state, .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, .ndo_setup_tc = nsim_setup_tc, .ndo_set_features = nsim_set_features, .ndo_get_iflink = nsim_get_iflink, .ndo_bpf = nsim_bpf, .ndo_open = nsim_open, .ndo_stop = nsim_stop, .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, .net_shaper_ops = &nsim_shaper_ops,};
static const struct net_device_ops nsim_vf_netdev_ops = { .ndo_start_xmit = nsim_start_xmit_vf, .ndo_set_rx_mode_async = nsim_set_rx_mode, .ndo_set_mac_address = eth_mac_addr, .ndo_validate_addr = eth_validate_addr, .ndo_change_mtu = nsim_change_mtu, .ndo_setup_tc = nsim_setup_tc, .ndo_set_features = nsim_set_features, .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid,};
/* We don't have true per-queue stats, yet, so do some random fakery here. * Only report stuff for queue 0. */static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, struct netdev_queue_stats_rx *stats){ struct rtnl_link_stats64 rtstats = {};
if (!idx) dev_get_stats(dev, &rtstats);
stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; stats->bytes = rtstats.rx_bytes;}
static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, struct netdev_queue_stats_tx *stats){ struct rtnl_link_stats64 rtstats = {};
if (!idx) dev_get_stats(dev, &rtstats);
stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; stats->bytes = rtstats.tx_bytes;}
static void nsim_get_base_stats(struct net_device *dev, struct netdev_queue_stats_rx *rx, struct netdev_queue_stats_tx *tx){ struct rtnl_link_stats64 rtstats = {};
dev_get_stats(dev, &rtstats);
rx->packets = !!rtstats.rx_packets; rx->bytes = 0; tx->packets = !!rtstats.tx_packets; tx->bytes = 0;}
static const struct netdev_stat_ops nsim_stat_ops = { .get_queue_stats_tx = nsim_get_queue_stats_tx, .get_queue_stats_rx = nsim_get_queue_stats_rx, .get_base_stats = nsim_get_base_stats,};
static struct nsim_rq *nsim_queue_alloc(void){ struct nsim_rq *rq;
rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT); if (!rq) return NULL;
skb_queue_head_init(&rq->skb_queue); nsim_rq_timer_init(rq); return rq;}
static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq){ hrtimer_cancel(&rq->napi_timer);
if (rq->skb_queue.qlen) { local_bh_disable(); dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); local_bh_enable(); }
skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); kfree(rq);}
/* Queue reset mode is controlled by ns->rq_reset_mode. * - normal - new NAPI new pool (old NAPI enabled when new added) * - mode 1 - allocate new pool (NAPI is only disabled / enabled) * - mode 2 - new NAPI new pool (old NAPI removed before new added) * - mode 3 - new NAPI new pool (old NAPI disabled when new added) */struct nsim_queue_mem { struct nsim_rq *rq; struct page_pool *pp;};
static intnsim_queue_mem_alloc(struct net_device *dev, struct netdev_queue_config *qcfg, void *per_queue_mem, int idx){ struct nsim_queue_mem *qmem = per_queue_mem; struct netdevsim *ns = netdev_priv(dev); int err;
if (ns->rq_reset_mode > 3) return -EINVAL;
if (ns->rq_reset_mode == 1) { if (!netif_running(ns->netdev)) return -ENETDOWN; return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); }
qmem->rq = nsim_queue_alloc(); if (!qmem->rq) return -ENOMEM;
err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); if (err) goto err_free;
if (!ns->rq_reset_mode) netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, idx);
return 0;
err_free: nsim_queue_free(dev, qmem->rq); return err;}
static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem){ struct nsim_queue_mem *qmem = per_queue_mem; struct netdevsim *ns = netdev_priv(dev);
page_pool_destroy(qmem->pp); if (qmem->rq) { if (!ns->rq_reset_mode) netif_napi_del_locked(&qmem->rq->napi); page_pool_destroy(qmem->rq->page_pool); nsim_queue_free(dev, qmem->rq); }}
static intnsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, void *per_queue_mem, int idx){ struct nsim_queue_mem *qmem = per_queue_mem; struct netdevsim *ns = netdev_priv(dev);
netdev_assert_locked(dev);
if (ns->rq_reset_mode == 1) { ns->rq[idx]->page_pool = qmem->pp; napi_enable_locked(&ns->rq[idx]->napi); return 0; }
/* netif_napi_add()/_del() should normally be called from alloc/free, * here we want to test various call orders. */ if (ns->rq_reset_mode == 2) { netif_napi_del_locked(&ns->rq[idx]->napi); netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, idx); } else if (ns->rq_reset_mode == 3) { netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, idx); netif_napi_del_locked(&ns->rq[idx]->napi); }
ns->rq[idx] = qmem->rq; netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi); napi_enable_locked(&ns->rq[idx]->napi);
return 0;}
static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx){ struct nsim_queue_mem *qmem = per_queue_mem; struct netdevsim *ns = netdev_priv(dev);
netdev_assert_locked(dev);
napi_disable_locked(&ns->rq[idx]->napi);
if (ns->rq_reset_mode == 1) { qmem->pp = ns->rq[idx]->page_pool; page_pool_disable_direct_recycling(qmem->pp); } else { qmem->rq = ns->rq[idx]; }
return 0;}
static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), .ndo_queue_mem_alloc = nsim_queue_mem_alloc, .ndo_queue_mem_free = nsim_queue_mem_free, .ndo_queue_start = nsim_queue_start, .ndo_queue_stop = nsim_queue_stop,};
static ssize_tnsim_qreset_write(struct file *file, const char __user *data, size_t count, loff_t *ppos){ struct netdevsim *ns = file->private_data; unsigned int queue, mode; char buf[32]; ssize_t ret;
if (count >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, data, count)) return -EFAULT; buf[count] = '\0';
ret = sscanf(buf, "%u %u", &queue, &mode); if (ret != 2) return -EINVAL;
netdev_lock(ns->netdev); if (queue >= ns->netdev->real_num_rx_queues) { ret = -EINVAL; goto exit_unlock; }
ns->rq_reset_mode = mode; ret = netdev_rx_queue_restart(ns->netdev, queue); ns->rq_reset_mode = 0; if (ret) goto exit_unlock;
ret = count;exit_unlock: netdev_unlock(ns->netdev); return ret;}
static const struct file_operations nsim_qreset_fops = { .open = simple_open, .write = nsim_qreset_write, .owner = THIS_MODULE,};
static ssize_tnsim_pp_hold_read(struct file *file, char __user *data, size_t count, loff_t *ppos){ struct netdevsim *ns = file->private_data; char buf[3] = "n\n";
if (ns->page) buf[0] = 'y';
return simple_read_from_buffer(data, count, ppos, buf, 2);}
static ssize_tnsim_pp_hold_write(struct file *file, const char __user *data, size_t count, loff_t *ppos){ struct netdevsim *ns = file->private_data; ssize_t ret; bool val;
ret = kstrtobool_from_user(data, count, &val); if (ret) return ret;
rtnl_lock(); ret = count; if (val == !!ns->page) goto exit;
if (!netif_running(ns->netdev) && val) { ret = -ENETDOWN; } else if (val) { ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); if (!ns->page) ret = -ENOMEM; } else { page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, ns->page, false); ns->page = NULL; }
exit: rtnl_unlock(); return ret;}
static const struct file_operations nsim_pp_hold_fops = { .open = simple_open, .read = nsim_pp_hold_read, .write = nsim_pp_hold_write, .llseek = generic_file_llseek, .owner = THIS_MODULE,};
static int nsim_vlan_show(struct seq_file *s, void *data){ struct netdevsim *ns = s->private; int vid;
for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) seq_printf(s, "ctag %d\n", vid); for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) seq_printf(s, "stag %d\n", vid);
return 0;}DEFINE_SHOW_ATTRIBUTE(nsim_vlan);
static void nsim_setup(struct net_device *dev){ ether_setup(dev); eth_hw_addr_random(dev);
dev->flags &= ~IFF_MULTICAST; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->features |= NETIF_F_HIGHDMA | NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | NETIF_F_LRO | NETIF_F_TSO | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER; dev->hw_features |= NETIF_F_HW_TC | NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | NETIF_F_LRO | NETIF_F_TSO | NETIF_F_LOOPBACK | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER; dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; dev->max_mtu = ETH_MAX_MTU; dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD;}
static int nsim_queue_init(struct netdevsim *ns){ struct net_device *dev = ns->netdev; int i;
ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT); if (!ns->rq) return -ENOMEM;
for (i = 0; i < dev->num_rx_queues; i++) { ns->rq[i] = nsim_queue_alloc(); if (!ns->rq[i]) goto err_free_prev; }
return 0;
err_free_prev: while (i--) kfree(ns->rq[i]); kfree(ns->rq); return -ENOMEM;}
static void nsim_queue_uninit(struct netdevsim *ns){ struct net_device *dev = ns->netdev; int i;
for (i = 0; i < dev->num_rx_queues; i++) nsim_queue_free(dev, ns->rq[i]);
kfree(ns->rq); ns->rq = NULL;}
static int nsim_init_netdevsim(struct netdevsim *ns){ struct netdevsim *peer; struct mock_phc *phc; int err;
phc = mock_phc_create(&ns->nsim_bus_dev->dev); if (IS_ERR(phc)) return PTR_ERR(phc);
ns->phc = phc; ns->netdev->netdev_ops = &nsim_netdev_ops; ns->netdev->stat_ops = &nsim_stat_ops; ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; netdev_lockdep_set_classes(ns->netdev);
err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); if (err) goto err_phc_destroy;
rtnl_lock(); err = nsim_queue_init(ns); if (err) goto err_utn_destroy;
err = nsim_bpf_init(ns); if (err) goto err_rq_destroy;
nsim_macsec_init(ns); nsim_ipsec_init(ns);
err = register_netdevice(ns->netdev); if (err) goto err_ipsec_teardown; rtnl_unlock();
err = nsim_psp_init(ns); if (err) goto err_unregister_netdev;
if (IS_ENABLED(CONFIG_DEBUG_NET)) { ns->nb.notifier_call = netdev_debug_event; if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, &ns->nn)) ns->nb.notifier_call = NULL; }
return 0;
err_unregister_netdev: rtnl_lock(); peer = rtnl_dereference(ns->peer); if (peer) RCU_INIT_POINTER(peer->peer, NULL); RCU_INIT_POINTER(ns->peer, NULL); unregister_netdevice(ns->netdev);err_ipsec_teardown: nsim_ipsec_teardown(ns); nsim_macsec_teardown(ns); nsim_bpf_uninit(ns);err_rq_destroy: nsim_queue_uninit(ns);err_utn_destroy: rtnl_unlock(); nsim_udp_tunnels_info_destroy(ns->netdev);err_phc_destroy: mock_phc_destroy(ns->phc); return err;}
static int nsim_init_netdevsim_vf(struct netdevsim *ns){ int err;
ns->netdev->netdev_ops = &nsim_vf_netdev_ops; rtnl_lock(); err = register_netdevice(ns->netdev); rtnl_unlock(); return err;}
static void nsim_exit_netdevsim(struct netdevsim *ns){ nsim_udp_tunnels_info_destroy(ns->netdev); mock_phc_destroy(ns->phc);}
struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port, u8 perm_addr[ETH_ALEN]){ struct net_device *dev; struct netdevsim *ns; int err;
dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, nsim_dev->nsim_bus_dev->num_queues); if (!dev) return ERR_PTR(-ENOMEM);
if (perm_addr) memcpy(dev->perm_addr, perm_addr, ETH_ALEN);
dev_net_set(dev, nsim_dev_net(nsim_dev)); ns = netdev_priv(dev); ns->netdev = dev; ns->nsim_dev = nsim_dev; ns->nsim_dev_port = nsim_dev_port; ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); nsim_ethtool_init(ns); if (nsim_dev_port_is_pf(nsim_dev_port)) err = nsim_init_netdevsim(ns); else err = nsim_init_netdevsim_vf(ns); if (err) goto err_free_netdev;
ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, ns, &nsim_pp_hold_fops); ns->qr_dfs = debugfs_create_file("queue_reset", 0200, nsim_dev_port->ddir, ns, &nsim_qreset_fops); ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir, ns, &nsim_vlan_fops); return ns;
err_free_netdev: nsim_ethtool_fini(ns); free_netdev(dev); return ERR_PTR(err);}
void nsim_destroy(struct netdevsim *ns){ struct net_device *dev = ns->netdev; struct netdevsim *peer; u16 vid;
debugfs_remove(ns->vlan_dfs); debugfs_remove(ns->qr_dfs); debugfs_remove(ns->pp_dfs); nsim_ethtool_fini(ns);
if (ns->nb.notifier_call) unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, &ns->nn);
if (nsim_dev_port_is_pf(ns->nsim_dev_port)) nsim_psp_uninit(ns);
rtnl_lock(); peer = rtnl_dereference(ns->peer); if (peer) RCU_INIT_POINTER(peer->peer, NULL); RCU_INIT_POINTER(ns->peer, NULL); unregister_netdevice(dev); if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { nsim_macsec_teardown(ns); nsim_ipsec_teardown(ns); nsim_bpf_uninit(ns); nsim_queue_uninit(ns); } rtnl_unlock(); if (nsim_dev_port_is_pf(ns->nsim_dev_port)) nsim_exit_netdevsim(ns);
for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) WARN_ON_ONCE(1); for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) WARN_ON_ONCE(1);
/* Put this intentionally late to exercise the orphaning path */ if (ns->page) { page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, ns->page, false); ns->page = NULL; }
free_netdev(dev);}
bool netdev_is_nsim(struct net_device *dev){ return dev->netdev_ops == &nsim_netdev_ops;}
static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack){ NL_SET_ERR_MSG_MOD(extack, "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); return -EOPNOTSUPP;}
static struct rtnl_link_ops nsim_link_ops __read_mostly = { .kind = DRV_NAME, .validate = nsim_validate,};
static int __init nsim_module_init(void){ int err;
err = nsim_dev_init(); if (err) return err;
err = nsim_bus_init(); if (err) goto err_dev_exit;
err = rtnl_link_register(&nsim_link_ops); if (err) goto err_bus_exit;
return 0;
err_bus_exit: nsim_bus_exit();err_dev_exit: nsim_dev_exit(); return err;}
static void __exit nsim_module_exit(void){ rtnl_link_unregister(&nsim_link_ops); nsim_bus_exit(); nsim_dev_exit();}
module_init(nsim_module_init);module_exit(nsim_module_exit);MODULE_LICENSE("GPL");MODULE_DESCRIPTION("Simulated networking device for testing");MODULE_ALIAS_RTNL_LINK(DRV_NAME);