Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2023 Isovalent */
#include <linux/netdevice.h>#include <linux/ethtool.h>#include <linux/etherdevice.h>#include <linux/filter.h>#include <linux/netfilter_netdev.h>#include <linux/bpf_mprog.h>#include <linux/indirect_call_wrapper.h>
#include <net/netdev_lock.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/xdp_sock_drv.h>#include <net/netkit.h>#include <net/dst.h>#include <net/tcx.h>
#define NETKIT_DRV_NAME "netkit"
#define NETKIT_NUM_RX_QUEUES_MAX 1024#define NETKIT_NUM_TX_QUEUES_MAX 1
#define NETKIT_NUM_RX_QUEUES_REAL 1#define NETKIT_NUM_TX_QUEUES_REAL 1
struct netkit { __cacheline_group_begin(netkit_fastpath); struct net_device __rcu *peer; struct bpf_mprog_entry __rcu *active; enum netkit_action policy; enum netkit_scrub scrub; struct bpf_mprog_bundle bundle; __cacheline_group_end(netkit_fastpath);
__cacheline_group_begin(netkit_slowpath); enum netkit_mode mode; enum netkit_pairing pair; bool primary; u32 headroom; __cacheline_group_end(netkit_slowpath);};
struct netkit_link { struct bpf_link link; struct net_device *dev;};
static struct rtnl_link_ops netkit_link_ops;
static __always_inline intnetkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, enum netkit_action ret){ const struct bpf_mprog_fp *fp; const struct bpf_prog *prog;
bpf_mprog_foreach_prog(entry, fp, prog) { bpf_compute_data_pointers(skb); ret = bpf_prog_run(prog, skb); if (ret != NETKIT_NEXT) break; } return ret;}
static void netkit_xnet(struct sk_buff *skb){ skb->priority = 0; skb->mark = 0;}
static void netkit_prep_forward(struct sk_buff *skb, bool xnet, bool xnet_scrub){ skb_scrub_packet(skb, false); nf_skip_egress(skb, true); skb_reset_mac_header(skb); if (!xnet) return; skb_clear_tstamp(skb); if (xnet_scrub) netkit_xnet(skb);}
static struct netkit *netkit_priv(const struct net_device *dev){ return netdev_priv(dev);}
static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev){ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; struct netkit *nk = netkit_priv(dev); enum netkit_action ret = READ_ONCE(nk->policy); netdev_tx_t ret_dev = NET_XMIT_SUCCESS; const struct bpf_mprog_entry *entry; struct net_device *peer; int len = skb->len;
bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); rcu_read_lock(); peer = rcu_dereference(nk->peer); if (unlikely(!peer || !(peer->flags & IFF_UP) || !pskb_may_pull(skb, ETH_HLEN) || skb_orphan_frags(skb, GFP_ATOMIC))) goto drop; netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), nk->scrub); eth_skb_pkt_type(skb, peer); skb->dev = peer; entry = rcu_dereference(nk->active); if (entry) ret = netkit_run(entry, skb, ret); switch (ret) { case NETKIT_NEXT: case NETKIT_PASS: eth_skb_pull_mac(skb); skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { dev_sw_netstats_tx_add(dev, 1, len); dev_sw_netstats_rx_add(peer, len); } else { goto drop_stats; } break; case NETKIT_REDIRECT: dev_sw_netstats_tx_add(dev, 1, len); skb_do_redirect(skb); break; case NETKIT_DROP: default:drop: kfree_skb(skb);drop_stats: dev_core_stats_tx_dropped_inc(dev); ret_dev = NET_XMIT_DROP; break; } rcu_read_unlock(); bpf_net_ctx_clear(bpf_net_ctx); return ret_dev;}
static int netkit_open(struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
if (nk->pair == NETKIT_DEVICE_SINGLE) { netif_carrier_on(dev); return 0; } if (!peer) return -ENOTCONN; if (peer->flags & IFF_UP) { netif_carrier_on(dev); netif_carrier_on(peer); } return 0;}
static int netkit_close(struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
netif_carrier_off(dev); if (peer) netif_carrier_off(peer); return 0;}
static int netkit_get_iflink(const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer; int iflink = 0;
rcu_read_lock(); peer = rcu_dereference(nk->peer); if (peer) iflink = READ_ONCE(peer->ifindex); rcu_read_unlock(); return iflink;}
static int netkit_set_multicast(struct net_device *dev, struct netdev_hw_addr_list *uc, struct netdev_hw_addr_list *mc){ /* Nothing to do, we receive whatever gets pushed to us! */ return 0;}
static int netkit_set_macaddr(struct net_device *dev, void *sa){ struct netkit *nk = netkit_priv(dev);
if (nk->mode != NETKIT_L2) return -EOPNOTSUPP;
return eth_mac_addr(dev, sa);}
static void netkit_set_headroom(struct net_device *dev, int headroom){ struct netkit *nk = netkit_priv(dev), *nk2; struct net_device *peer;
if (headroom < 0) headroom = NET_SKB_PAD;
rcu_read_lock(); peer = rcu_dereference(nk->peer); if (!peer) { nk->headroom = headroom; dev->needed_headroom = headroom; } else { nk2 = netkit_priv(peer); nk->headroom = headroom; headroom = max(nk->headroom, nk2->headroom);
peer->needed_headroom = headroom; dev->needed_headroom = headroom; } rcu_read_unlock();}
INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev){ return rcu_dereference(netkit_priv(dev)->peer);}
static void netkit_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats){ dev_fetch_sw_netstats(stats, dev->tstats); stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);}
static bool netkit_xsk_supported_at_phys(const struct net_device *dev){ if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xdp_xmit || !dev->netdev_ops->ndo_xsk_wakeup) return false; return true;}
static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp){ struct netkit *nk = netkit_priv(dev); struct netdev_bpf xdp_lower; struct netdev_rx_queue *rxq; struct net_device *phys; bool create = false; int ret = -EBUSY;
switch (xdp->command) { case XDP_SETUP_XSK_POOL: if (nk->pair == NETKIT_DEVICE_PAIR) return -EOPNOTSUPP; if (xdp->xsk.queue_id >= dev->real_num_rx_queues) return -EINVAL;
rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id); if (!rxq->lease) return -EOPNOTSUPP;
phys = rxq->lease->dev; if (!netkit_xsk_supported_at_phys(phys)) return -EOPNOTSUPP;
create = xdp->xsk.pool; memcpy(&xdp_lower, xdp, sizeof(xdp_lower)); xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease); break; case XDP_SETUP_PROG: return -EOPNOTSUPP; default: return -EINVAL; }
netdev_lock(phys); if (create && (phys->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) { ret = -EOPNOTSUPP; goto out; } if (!create || !dev_get_min_mp_channel_count(phys)) ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower);out: netdev_unlock(phys); return ret;}
static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags){ struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *phys;
if (queue_id >= dev->real_num_rx_queues) return -EINVAL;
rxq = __netif_get_rx_queue(dev, queue_id); rxq_lease = READ_ONCE(rxq->lease); if (unlikely(!rxq_lease)) return -EOPNOTSUPP;
/* netkit_xsk already validated full xsk support, hence it's * fine to call into ndo_xsk_wakeup right away given this * was a prerequisite to get here in the first place. The * phys xsk support cannot change without tearing down the * device (which clears the lease first). */ phys = rxq_lease->dev; return phys->netdev_ops->ndo_xsk_wakeup(phys, get_netdev_rx_queue_index(rxq_lease), flags);}
static int netkit_init(struct net_device *dev){ netdev_lockdep_set_classes(dev); return 0;}
static void netkit_uninit(struct net_device *dev);
static const struct net_device_ops netkit_netdev_ops = { .ndo_init = netkit_init, .ndo_open = netkit_open, .ndo_stop = netkit_close, .ndo_start_xmit = netkit_xmit, .ndo_set_rx_mode_async = netkit_set_multicast, .ndo_set_rx_headroom = netkit_set_headroom, .ndo_set_mac_address = netkit_set_macaddr, .ndo_get_iflink = netkit_get_iflink, .ndo_get_peer_dev = netkit_peer_dev, .ndo_get_stats64 = netkit_get_stats, .ndo_uninit = netkit_uninit, .ndo_bpf = netkit_xsk, .ndo_xsk_wakeup = netkit_xsk_wakeup, .ndo_features_check = passthru_features_check,};
static void netkit_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info){ strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver));}
static const struct ethtool_ops netkit_ethtool_ops = { .get_drvinfo = netkit_get_drvinfo,};
static int netkit_queue_create(struct net_device *dev, struct netlink_ext_ack *extack){ struct netkit *nk = netkit_priv(dev); u32 rxq_count_old, rxq_count_new; int err;
rxq_count_old = dev->real_num_rx_queues; rxq_count_new = rxq_count_old + 1;
/* In paired mode, only the non-primary (peer) device can * create leased queues since the primary is the management * side. In single device mode, leasing is always allowed. */ if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) { NL_SET_ERR_MSG(extack, "netkit can only lease against the peer device"); return -EOPNOTSUPP; }
err = netif_set_real_num_rx_queues(dev, rxq_count_new); if (err) { if (rxq_count_new > dev->num_rx_queues) NL_SET_ERR_MSG(extack, "netkit maximum queue limit reached"); else NL_SET_ERR_MSG_FMT(extack, "netkit cannot create more queues err=%d", err); return err; }
return rxq_count_old;}
static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = { .ndo_queue_create = netkit_queue_create,};
static struct net_device *netkit_alloc(struct nlattr *tb[], const char *ifname, unsigned char name_assign_type, unsigned int num_tx_queues, unsigned int num_rx_queues){ const struct rtnl_link_ops *ops = &netkit_link_ops; struct net_device *dev;
if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX || num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX) return ERR_PTR(-EOPNOTSUPP);
dev = alloc_netdev_mqs(ops->priv_size, ifname, name_assign_type, ops->setup, num_tx_queues, num_rx_queues); if (dev) { dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL; dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL; } return dev;}
static void netkit_queue_unlease(struct net_device *dev){ struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *dev_lease; int i;
if (dev->real_num_rx_queues == 1) return;
netdev_lock(dev); for (i = 1; i < dev->real_num_rx_queues; i++) { rxq = __netif_get_rx_queue(dev, i); rxq_lease = rxq->lease; dev_lease = rxq_lease->dev;
netdev_lock(dev_lease); netdev_rx_queue_unlease(rxq, rxq_lease); netdev_unlock(dev_lease); } netdev_unlock(dev);}
static void netkit_setup(struct net_device *dev){ static const netdev_features_t netkit_features_hw_vlan = NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX; static const netdev_features_t netkit_features = netkit_features_hw_vlan | NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL;
ether_setup(dev); dev->max_mtu = ETH_MAX_MTU; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
dev->flags |= IFF_NOARP; dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->priv_flags |= IFF_PHONY_HEADROOM; dev->priv_flags |= IFF_NO_QUEUE; dev->priv_flags |= IFF_DISABLE_NETPOLL; dev->lltx = true; dev->netmem_tx = NETMEM_TX_NO_DMA;
dev->netdev_ops = &netkit_netdev_ops; dev->ethtool_ops = &netkit_ethtool_ops; dev->queue_mgmt_ops = &netkit_queue_mgmt_ops;
dev->features |= netkit_features; dev->hw_features = netkit_features; dev->hw_enc_features = netkit_features; dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
dev->needs_free_netdev = true;
netif_set_tso_max_size(dev, GSO_MAX_SIZE);}
static struct net *netkit_get_link_net(const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
return peer ? dev_net(peer) : dev_net(dev);}
static int netkit_check_policy(int policy, struct nlattr *tb, struct netlink_ext_ack *extack){ switch (policy) { case NETKIT_PASS: case NETKIT_DROP: return 0; default: NL_SET_ERR_MSG_ATTR(extack, tb, "Provided default xmit policy not supported"); return -EINVAL; }}
static int netkit_validate(struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack){ struct nlattr *attr = tb[IFLA_ADDRESS];
if (!attr) return 0; if (nla_len(attr) != ETH_ALEN) return -EINVAL; if (!is_valid_ether_addr(nla_data(attr))) return -EADDRNOTAVAIL; return 0;}
static int netkit_new_link(struct net_device *dev, struct rtnl_newlink_params *params, struct netlink_ext_ack *extack){ struct net *peer_net = rtnl_newlink_peer_net(params); enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT; enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT; struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr; enum netkit_pairing pair = NETKIT_DEVICE_PAIR; enum netkit_action policy_prim = NETKIT_PASS; enum netkit_action policy_peer = NETKIT_PASS; bool seen_peer = false, seen_scrub = false; struct nlattr **data = params->data; enum netkit_mode mode = NETKIT_L3; unsigned char ifname_assign_type; struct nlattr **tb = params->tb; u16 headroom = 0, tailroom = 0; struct ifinfomsg *ifmp = NULL; struct net_device *peer = NULL; char ifname[IFNAMSIZ]; struct netkit *nk; int err;
tbp = tb; if (data) { if (data[IFLA_NETKIT_MODE]) mode = nla_get_u32(data[IFLA_NETKIT_MODE]); if (data[IFLA_NETKIT_PEER_INFO]) { attr = data[IFLA_NETKIT_PEER_INFO]; ifmp = nla_data(attr); rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack); tbp = peer_tb; } if (data[IFLA_NETKIT_SCRUB]) scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]); if (data[IFLA_NETKIT_PEER_SCRUB]) scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]); if (data[IFLA_NETKIT_POLICY]) { attr = data[IFLA_NETKIT_POLICY]; policy_prim = nla_get_u32(attr); err = netkit_check_policy(policy_prim, attr, extack); if (err < 0) return err; } if (data[IFLA_NETKIT_PEER_POLICY]) { attr = data[IFLA_NETKIT_PEER_POLICY]; policy_peer = nla_get_u32(attr); err = netkit_check_policy(policy_peer, attr, extack); if (err < 0) return err; } if (data[IFLA_NETKIT_HEADROOM]) headroom = nla_get_u16(data[IFLA_NETKIT_HEADROOM]); if (data[IFLA_NETKIT_TAILROOM]) tailroom = nla_get_u16(data[IFLA_NETKIT_TAILROOM]); if (data[IFLA_NETKIT_PAIRING]) pair = nla_get_u32(data[IFLA_NETKIT_PAIRING]);
seen_scrub = data[IFLA_NETKIT_SCRUB]; seen_peer = data[IFLA_NETKIT_PEER_INFO] || data[IFLA_NETKIT_PEER_SCRUB] || data[IFLA_NETKIT_PEER_POLICY]; }
if (ifmp && tbp[IFLA_IFNAME]) { nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); ifname_assign_type = NET_NAME_USER; } else { strscpy(ifname, "nk%d", IFNAMSIZ); ifname_assign_type = NET_NAME_ENUM; } if (mode != NETKIT_L2 && (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS])) return -EOPNOTSUPP; if (pair == NETKIT_DEVICE_SINGLE && (tb != tbp || seen_peer || seen_scrub || policy_prim != NETKIT_PASS)) return -EOPNOTSUPP;
if (pair == NETKIT_DEVICE_PAIR) { peer = rtnl_create_link(peer_net, ifname, ifname_assign_type, &netkit_link_ops, tbp, extack); if (IS_ERR(peer)) return PTR_ERR(peer);
netif_inherit_tso_max(peer, dev); if (headroom) peer->needed_headroom = headroom; if (tailroom) peer->needed_tailroom = tailroom; if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS])) eth_hw_addr_random(peer); if (ifmp && dev->ifindex) peer->ifindex = ifmp->ifi_index;
nk = netkit_priv(peer); nk->primary = false; nk->policy = policy_peer; nk->scrub = scrub_peer; nk->mode = mode; nk->pair = pair; nk->headroom = headroom; bpf_mprog_bundle_init(&nk->bundle);
err = register_netdevice(peer); if (err < 0) goto err_register_peer; netif_carrier_off(peer); if (mode == NETKIT_L2) dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
err = rtnl_configure_link(peer, NULL, 0, NULL); if (err < 0) goto err_configure_peer; }
if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS]) eth_hw_addr_random(dev); if (tb[IFLA_IFNAME]) nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); else strscpy(dev->name, "nk%d", IFNAMSIZ); if (headroom) dev->needed_headroom = headroom; if (tailroom) dev->needed_tailroom = tailroom;
nk = netkit_priv(dev); nk->primary = true; nk->policy = policy_prim; nk->scrub = scrub_prim; nk->mode = mode; nk->pair = pair; nk->headroom = headroom; bpf_mprog_bundle_init(&nk->bundle);
if (pair == NETKIT_DEVICE_SINGLE) xdp_set_features_flag(dev, NETDEV_XDP_ACT_XSK);
err = register_netdevice(dev); if (err < 0) goto err_configure_peer; netif_carrier_off(dev); if (mode == NETKIT_L2) dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
rcu_assign_pointer(netkit_priv(dev)->peer, peer); if (peer) rcu_assign_pointer(netkit_priv(peer)->peer, dev); return 0;err_configure_peer: if (peer) unregister_netdevice(peer); return err;err_register_peer: free_netdev(peer); return err;}
static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev, bool bundle_fallback){ struct netkit *nk = netkit_priv(dev); struct bpf_mprog_entry *entry;
ASSERT_RTNL(); entry = rcu_dereference_rtnl(nk->active); if (entry) return entry; if (bundle_fallback) return &nk->bundle.a; return NULL;}
static void netkit_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry){ struct netkit *nk = netkit_priv(dev);
ASSERT_RTNL(); rcu_assign_pointer(nk->active, entry);}
static void netkit_entry_sync(void){ synchronize_rcu();}
static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which){ struct net_device *dev; struct netkit *nk;
ASSERT_RTNL();
switch (which) { case BPF_NETKIT_PRIMARY: case BPF_NETKIT_PEER: break; default: return ERR_PTR(-EINVAL); }
dev = __dev_get_by_index(net, ifindex); if (!dev) return ERR_PTR(-ENODEV); if (dev->netdev_ops != &netkit_netdev_ops) return ERR_PTR(-ENXIO);
nk = netkit_priv(dev); if (!nk->primary) return ERR_PTR(-EACCES); if (nk->pair == NETKIT_DEVICE_SINGLE) return ERR_PTR(-EOPNOTSUPP); if (which == BPF_NETKIT_PEER) { dev = rcu_dereference_rtnl(nk->peer); if (!dev) return ERR_PTR(-ENODEV); } return dev;}
int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog){ struct bpf_mprog_entry *entry, *entry_new; struct bpf_prog *replace_prog = NULL; struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, attr->attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } entry = netkit_entry_fetch(dev, true); if (attr->attach_flags & BPF_F_REPLACE) { replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, prog->type); if (IS_ERR(replace_prog)) { ret = PTR_ERR(replace_prog); replace_prog = NULL; goto out; } } ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, attr->attach_flags, attr->relative_fd, attr->expected_revision); if (!ret) { if (entry != entry_new) { netkit_entry_update(dev, entry_new); netkit_entry_sync(); } bpf_mprog_commit(entry); }out: if (replace_prog) bpf_prog_put(replace_prog); rtnl_unlock(); return ret;}
int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog){ struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, attr->attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } entry = netkit_entry_fetch(dev, false); if (!entry) { ret = -ENOENT; goto out; } ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, attr->relative_fd, attr->expected_revision); if (!ret) { if (!bpf_mprog_total(entry_new)) entry_new = NULL; netkit_entry_update(dev, entry_new); netkit_entry_sync(); bpf_mprog_commit(entry); }out: rtnl_unlock(); return ret;}
int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr){ struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->query.target_ifindex, attr->query.attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));out: rtnl_unlock(); return ret;}
static struct netkit_link *netkit_link(const struct bpf_link *link){ return container_of(link, struct netkit_link, link);}
static int netkit_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd, u64 revision){ struct netkit_link *nkl = netkit_link(link); struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev = nkl->dev; int ret;
ASSERT_RTNL(); entry = netkit_entry_fetch(dev, true); ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, id_or_fd, revision); if (!ret) { if (entry != entry_new) { netkit_entry_update(dev, entry_new); netkit_entry_sync(); } bpf_mprog_commit(entry); } return ret;}
static void netkit_link_release(struct bpf_link *link){ struct netkit_link *nkl = netkit_link(link); struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev; int ret = 0;
rtnl_lock(); dev = nkl->dev; if (!dev) goto out; entry = netkit_entry_fetch(dev, false); if (!entry) { ret = -ENOENT; goto out; } ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); if (!ret) { if (!bpf_mprog_total(entry_new)) entry_new = NULL; netkit_entry_update(dev, entry_new); netkit_entry_sync(); bpf_mprog_commit(entry); nkl->dev = NULL; }out: WARN_ON_ONCE(ret); rtnl_unlock();}
static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog, struct bpf_prog *oprog){ struct netkit_link *nkl = netkit_link(link); struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev; int ret = 0;
rtnl_lock(); dev = nkl->dev; if (!dev) { ret = -ENOLINK; goto out; } if (oprog && link->prog != oprog) { ret = -EPERM; goto out; } oprog = link->prog; if (oprog == nprog) { bpf_prog_put(nprog); goto out; } entry = netkit_entry_fetch(dev, false); if (!entry) { ret = -ENOENT; goto out; } ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, BPF_F_REPLACE | BPF_F_ID, link->prog->aux->id, 0); if (!ret) { WARN_ON_ONCE(entry != entry_new); oprog = xchg(&link->prog, nprog); bpf_prog_put(oprog); bpf_mprog_commit(entry); }out: rtnl_unlock(); return ret;}
static void netkit_link_dealloc(struct bpf_link *link){ kfree(netkit_link(link));}
static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq){ const struct netkit_link *nkl = netkit_link(link); u32 ifindex = 0;
rtnl_lock(); if (nkl->dev) ifindex = nkl->dev->ifindex; rtnl_unlock();
seq_printf(seq, "ifindex:\t%u\n", ifindex); seq_printf(seq, "attach_type:\t%u (%s)\n", link->attach_type, link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer");}
static int netkit_link_fill_info(const struct bpf_link *link, struct bpf_link_info *info){ const struct netkit_link *nkl = netkit_link(link); u32 ifindex = 0;
rtnl_lock(); if (nkl->dev) ifindex = nkl->dev->ifindex; rtnl_unlock();
info->netkit.ifindex = ifindex; info->netkit.attach_type = link->attach_type; return 0;}
static int netkit_link_detach(struct bpf_link *link){ netkit_link_release(link); return 0;}
static const struct bpf_link_ops netkit_link_lops = { .release = netkit_link_release, .detach = netkit_link_detach, .dealloc = netkit_link_dealloc, .update_prog = netkit_link_update, .show_fdinfo = netkit_link_fdinfo, .fill_link_info = netkit_link_fill_info,};
static int netkit_link_init(struct netkit_link *nkl, struct bpf_link_primer *link_primer, const union bpf_attr *attr, struct net_device *dev, struct bpf_prog *prog){ bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT, &netkit_link_lops, prog, attr->link_create.attach_type); nkl->dev = dev; return bpf_link_prime(&nkl->link, link_primer);}
int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog){ struct bpf_link_primer link_primer; struct netkit_link *nkl; struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->link_create.target_ifindex, attr->link_create.attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } nkl = kzalloc_obj(*nkl, GFP_KERNEL_ACCOUNT); if (!nkl) { ret = -ENOMEM; goto out; } ret = netkit_link_init(nkl, &link_primer, attr, dev, prog); if (ret) { kfree(nkl); goto out; } ret = netkit_link_prog_attach(&nkl->link, attr->link_create.flags, attr->link_create.netkit.relative_fd, attr->link_create.netkit.expected_revision); if (ret) { nkl->dev = NULL; bpf_link_cleanup(&link_primer); goto out; } ret = bpf_link_settle(&link_primer);out: rtnl_unlock(); return ret;}
static void netkit_release_all(struct net_device *dev){ struct bpf_mprog_entry *entry; struct bpf_tuple tuple = {}; struct bpf_mprog_fp *fp; struct bpf_mprog_cp *cp;
entry = netkit_entry_fetch(dev, false); if (!entry) return; netkit_entry_update(dev, NULL); netkit_entry_sync(); bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { if (tuple.link) netkit_link(tuple.link)->dev = NULL; else bpf_prog_put(tuple.prog); }}
static void netkit_uninit(struct net_device *dev){ netkit_release_all(dev); netkit_queue_unlease(dev);}
static void netkit_del_link(struct net_device *dev, struct list_head *head){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
RCU_INIT_POINTER(nk->peer, NULL); unregister_netdevice_queue(dev, head); if (peer) { nk = netkit_priv(peer); RCU_INIT_POINTER(nk->peer, NULL); /* Guard against the peer already being in an unregister * list (e.g. same-namespace teardown where the peer is * in the caller's dev_kill_list). list_move_tail() on an * already-queued device would otherwise corrupt that * list's iteration. This situation can occur via netkit * notifier, hence guard against this scenario. */ if (!unregister_netdevice_queued(peer)) unregister_netdevice_queue(peer, head); }}
static int netkit_change_link(struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); enum netkit_action policy; struct nlattr *attr; int err, i; static const struct { u32 attr; char *name; } fixed_params[] = { { IFLA_NETKIT_MODE, "operating mode" }, { IFLA_NETKIT_SCRUB, "scrubbing" }, { IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" }, { IFLA_NETKIT_PEER_INFO, "peer info" }, { IFLA_NETKIT_HEADROOM, "headroom" }, { IFLA_NETKIT_TAILROOM, "tailroom" }, { IFLA_NETKIT_PAIRING, "pairing" }, };
if (!nk->primary) { NL_SET_ERR_MSG(extack, "netkit link settings can be changed only through the primary device"); return -EACCES; }
for (i = 0; i < ARRAY_SIZE(fixed_params); i++) { attr = data[fixed_params[i].attr]; if (attr) { NL_SET_ERR_MSG_ATTR_FMT(extack, attr, "netkit link %s cannot be changed after device creation", fixed_params[i].name); return -EACCES; } }
if (data[IFLA_NETKIT_POLICY]) { err = -EOPNOTSUPP; attr = data[IFLA_NETKIT_POLICY]; policy = nla_get_u32(attr); if (nk->pair == NETKIT_DEVICE_PAIR) err = netkit_check_policy(policy, attr, extack); if (err) return err; WRITE_ONCE(nk->policy, policy); }
if (data[IFLA_NETKIT_PEER_POLICY]) { err = -EOPNOTSUPP; attr = data[IFLA_NETKIT_PEER_POLICY]; policy = nla_get_u32(attr); if (peer) err = netkit_check_policy(policy, attr, extack); if (err) return err; nk = netkit_priv(peer); WRITE_ONCE(nk->policy, policy); }
return 0;}
static void netkit_check_lease_unregister(struct net_device *dev){ LIST_HEAD(list_kill); u32 q_idx;
if (READ_ONCE(dev->reg_state) != NETREG_UNREGISTERING || !dev->dev.parent) return;
netdev_lock_ops(dev); for (q_idx = 0; q_idx < dev->real_num_rx_queues; q_idx++) { struct net_device *tmp = dev; struct netdev_rx_queue *rxq; u32 tmp_q_idx = q_idx;
rxq = __netif_get_rx_queue_lease(&tmp, &tmp_q_idx, NETIF_PHYS_TO_VIRT); if (rxq && tmp != dev && tmp->netdev_ops == &netkit_netdev_ops) { /* A single phys device can have multiple queues leased * to one netkit device. We can only queue that netkit * device once to the list_kill. Queues of that phys * device can be leased with different individual netkit * devices, hence we batch via list_kill. */ if (unregister_netdevice_queued(tmp)) continue; netkit_del_link(tmp, &list_kill); } } netdev_unlock_ops(dev); unregister_netdevice_many(&list_kill);}
static int netkit_notifier(struct notifier_block *this, unsigned long event, void *ptr){ struct net_device *dev = netdev_notifier_info_to_dev(ptr);
if (event == NETDEV_UNREGISTER) netkit_check_lease_unregister(dev); return NOTIFY_DONE;}
static size_t netkit_get_size(const struct net_device *dev){ return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */ nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */ nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */ nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PAIRING */ 0;}
static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev){ const struct netkit *nk = netkit_priv(dev); const struct net_device *peer;
if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary)) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_POLICY, READ_ONCE(nk->policy))) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode)) return -EMSGSIZE; if (nk->pair == NETKIT_DEVICE_PAIR && nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub)) return -EMSGSIZE; if (nla_put_u16(skb, IFLA_NETKIT_HEADROOM, dev->needed_headroom)) return -EMSGSIZE; if (nla_put_u16(skb, IFLA_NETKIT_TAILROOM, dev->needed_tailroom)) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_PAIRING, nk->pair)) return -EMSGSIZE;
rcu_read_lock(); peer = rcu_dereference(nk->peer); if (peer) { nk = netkit_priv(peer); if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, READ_ONCE(nk->policy)) || nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub)) { rcu_read_unlock(); return -EMSGSIZE; } } rcu_read_unlock();
return 0;}
static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = { [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) }, [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3), [IFLA_NETKIT_POLICY] = { .type = NLA_U32 }, [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 }, [IFLA_NETKIT_HEADROOM] = { .type = NLA_U16 }, [IFLA_NETKIT_TAILROOM] = { .type = NLA_U16 }, [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), [IFLA_NETKIT_PAIRING] = NLA_POLICY_MAX(NLA_U32, NETKIT_DEVICE_SINGLE), [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT, .reject_message = "Primary attribute is read-only" },};
static struct rtnl_link_ops netkit_link_ops = { .kind = NETKIT_DRV_NAME, .priv_size = sizeof(struct netkit), .alloc = netkit_alloc, .setup = netkit_setup, .newlink = netkit_new_link, .dellink = netkit_del_link, .changelink = netkit_change_link, .get_link_net = netkit_get_link_net, .get_size = netkit_get_size, .fill_info = netkit_fill_info, .policy = netkit_policy, .validate = netkit_validate, .peer_type = IFLA_NETKIT_PEER_INFO, .maxtype = IFLA_NETKIT_MAX,};
static struct notifier_block netkit_netdev_notifier = { .notifier_call = netkit_notifier,};
static __init int netkit_mod_init(void){ int ret;
BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT || (int)NETKIT_PASS != (int)TCX_PASS || (int)NETKIT_DROP != (int)TCX_DROP || (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
ret = rtnl_link_register(&netkit_link_ops); if (ret) return ret; ret = register_netdevice_notifier(&netkit_netdev_notifier); if (ret) rtnl_link_unregister(&netkit_link_ops); return ret;}
static __exit void netkit_mod_exit(void){ unregister_netdevice_notifier(&netkit_netdev_notifier); rtnl_link_unregister(&netkit_link_ops);}
module_init(netkit_mod_init);module_exit(netkit_mod_exit);
MODULE_DESCRIPTION("BPF-programmable network device");MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");MODULE_LICENSE("GPL");MODULE_ALIAS_RTNL_LINK(NETKIT_DRV_NAME);