Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656// SPDX-License-Identifier: GPL-2.0/* Multipath TCP * * Copyright (c) 2025, Matthieu Baerts. */
#define pr_fmt(fmt) "MPTCP: " fmt
#include <net/netns/generic.h>
#include "protocol.h"#include "mib.h"#include "mptcp_pm_gen.h"
static int pm_nl_pernet_id;
struct pm_nl_pernet { /* protects pernet updates */ spinlock_t lock; struct list_head endp_list; u8 endpoints; u8 endp_signal_max; u8 endp_subflow_max; u8 endp_laminar_max; u8 endp_fullmesh_max; u8 limit_add_addr_accepted; u8 limit_extra_subflows; u8 next_id; DECLARE_BITMAP(id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);};
#define MPTCP_PM_ADDR_MAX 8#define MPTCP_PM_SUBFLOWS_MAX 64
static struct pm_nl_pernet *pm_nl_get_pernet(const struct net *net){ return net_generic(net, pm_nl_pernet_id);}
static struct pm_nl_pernet *pm_nl_get_pernet_from_msk(const struct mptcp_sock *msk){ return pm_nl_get_pernet(sock_net((struct sock *)msk));}
static struct pm_nl_pernet *genl_info_pm_nl(struct genl_info *info){ return pm_nl_get_pernet(genl_info_net(info));}
u8 mptcp_pm_get_endp_signal_max(const struct mptcp_sock *msk){ const struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_signal_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_signal_max);
u8 mptcp_pm_get_endp_subflow_max(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_subflow_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_subflow_max);
u8 mptcp_pm_get_endp_laminar_max(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_laminar_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_laminar_max);
u8 mptcp_pm_get_endp_fullmesh_max(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_fullmesh_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_fullmesh_max);
u8 mptcp_pm_get_limit_add_addr_accepted(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->limit_add_addr_accepted);}EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_add_addr_accepted);
u8 mptcp_pm_get_limit_extra_subflows(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->limit_extra_subflows);}EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_extra_subflows);
static bool has_subflow_daddr(const struct mptcp_sock *msk, const struct mptcp_addr_info *daddr){ struct mptcp_subflow_context *subflow; struct mptcp_addr_info cur;
mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
if (!((1 << inet_sk_state_load(ssk)) & (TCPF_ESTABLISHED | TCPF_SYN_SENT | TCPF_SYN_RECV))) continue;
mptcp_remote_address((struct sock_common *)ssk, &cur); if (mptcp_addresses_equal(&cur, daddr, daddr->port)) return true; }
return false;}
static boolselect_local_address(const struct pm_nl_pernet *pernet, const struct mptcp_sock *msk, struct mptcp_pm_local *new_local){ struct mptcp_pm_addr_entry *entry; bool found = false;
msk_owned_by_me(msk);
rcu_read_lock(); list_for_each_entry_rcu(entry, &pernet->endp_list, list) { if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) continue;
if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap)) continue;
new_local->addr = entry->addr; new_local->flags = entry->flags; new_local->ifindex = entry->ifindex; found = true; break; } rcu_read_unlock();
return found;}
static boolselect_signal_address(struct pm_nl_pernet *pernet, const struct mptcp_sock *msk, struct mptcp_pm_local *new_local){ struct mptcp_pm_addr_entry *entry; bool found = false;
rcu_read_lock(); /* do not keep any additional per socket state, just signal * the address list in order. * Note: removal from the local address list during the msk life-cycle * can lead to additional addresses not being announced. */ list_for_each_entry_rcu(entry, &pernet->endp_list, list) { if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap)) continue;
if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL)) continue;
new_local->addr = entry->addr; new_local->flags = entry->flags; new_local->ifindex = entry->ifindex; found = true; break; } rcu_read_unlock();
return found;}
static unsigned intfill_remote_addr(struct mptcp_sock *msk, struct mptcp_addr_info *local, struct mptcp_addr_info *addrs){ bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0); struct mptcp_addr_info remote = { 0 }; struct sock *sk = (struct sock *)msk;
if (deny_id0) return 0;
mptcp_remote_address((struct sock_common *)sk, &remote);
if (!mptcp_pm_addr_families_match(sk, local, &remote)) return 0;
msk->pm.extra_subflows++; *addrs = remote;
return 1;}
static unsigned intfill_remote_addresses_fullmesh(struct mptcp_sock *msk, struct mptcp_addr_info *local, struct mptcp_addr_info *addrs, int addrs_size){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0); DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); struct sock *sk = (struct sock *)msk, *ssk; struct mptcp_subflow_context *subflow; int i = 0;
/* Forbid creation of new subflows matching existing ones, possibly * already created by incoming ADD_ADDR */ bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); mptcp_for_each_subflow(msk, subflow) if (READ_ONCE(subflow->local_id) == local->id) __set_bit(subflow->remote_id, unavail_id);
mptcp_for_each_subflow(msk, subflow) { ssk = mptcp_subflow_tcp_sock(subflow); mptcp_remote_address((struct sock_common *)ssk, &addrs[i]); addrs[i].id = READ_ONCE(subflow->remote_id); if (deny_id0 && !addrs[i].id) continue;
if (test_bit(addrs[i].id, unavail_id)) continue;
if (!mptcp_pm_addr_families_match(sk, local, &addrs[i])) continue;
/* forbid creating multiple address towards this id */ __set_bit(addrs[i].id, unavail_id); msk->pm.extra_subflows++; i++;
if (msk->pm.extra_subflows >= limit_extra_subflows || i == addrs_size) break; }
return i;}
/* Fill all the remote addresses into the array addrs[], * and return the array size. */static unsigned intfill_remote_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *local, bool fullmesh, struct mptcp_addr_info *addrs, int addrs_size){ /* Non-fullmesh: fill in the single entry corresponding to the primary * MPC subflow remote address, and return 1, corresponding to 1 entry. */ if (!fullmesh) return fill_remote_addr(msk, local, addrs);
/* Fullmesh endpoint: fill all possible remote addresses */ return fill_remote_addresses_fullmesh(msk, local, addrs, addrs_size);}
static struct mptcp_pm_addr_entry *__lookup_addr_by_id(struct pm_nl_pernet *pernet, unsigned int id){ struct mptcp_pm_addr_entry *entry;
list_for_each_entry_rcu(entry, &pernet->endp_list, list, lockdep_is_held(&pernet->lock)) { if (entry->addr.id == id) return entry; } return NULL;}
static struct mptcp_pm_addr_entry *__lookup_addr(struct pm_nl_pernet *pernet, const struct mptcp_addr_info *info){ struct mptcp_pm_addr_entry *entry;
list_for_each_entry_rcu(entry, &pernet->endp_list, list, lockdep_is_held(&pernet->lock)) { if (mptcp_addresses_equal(&entry->addr, info, entry->addr.port)) return entry; } return NULL;}
static u8 mptcp_endp_get_local_id(struct mptcp_sock *msk, const struct mptcp_addr_info *addr){ return msk->mpc_endpoint_id == addr->id ? 0 : addr->id;}
/* Set mpc_endpoint_id, and send MP_PRIO for ID0 if needed */static void mptcp_mpc_endpoint_setup(struct mptcp_sock *msk){ struct mptcp_subflow_context *subflow; struct mptcp_pm_addr_entry *entry; struct mptcp_addr_info mpc_addr; struct pm_nl_pernet *pernet; bool backup = false;
/* do lazy endpoint usage accounting for the MPC subflows */ if (likely(msk->pm.status & BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED)) || !msk->first) return;
subflow = mptcp_subflow_ctx(msk->first); pernet = pm_nl_get_pernet_from_msk(msk);
mptcp_local_address((struct sock_common *)msk->first, &mpc_addr); rcu_read_lock(); entry = __lookup_addr(pernet, &mpc_addr); if (entry) { __clear_bit(entry->addr.id, msk->pm.id_avail_bitmap); msk->mpc_endpoint_id = entry->addr.id; backup = !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP); } rcu_read_unlock();
/* Send MP_PRIO */ if (backup) mptcp_pm_send_ack(msk, subflow, true, backup);
msk->pm.status |= BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED);}
static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk); u8 endp_signal_max = mptcp_pm_get_endp_signal_max(msk); struct sock *sk = (struct sock *)msk; bool signal_and_subflow = false; struct mptcp_pm_local local;
mptcp_mpc_endpoint_setup(msk); if (!mptcp_is_fully_established(sk)) return;
pr_debug("local %d:%d signal %d:%d subflows %d:%d\n", msk->pm.local_addr_used, endp_subflow_max, msk->pm.add_addr_signaled, endp_signal_max, msk->pm.extra_subflows, limit_extra_subflows);
/* check first for announce */ if (msk->pm.add_addr_signaled < endp_signal_max) { u8 endp_id;
/* due to racing events on both ends we can reach here while * previous add address is still running: if we invoke now * mptcp_pm_announce_addr(), that will fail and the * corresponding id will be marked as used. * Instead let the PM machinery reschedule us when the * current address announce will be completed. */ if (msk->pm.addr_signal & BIT(MPTCP_ADD_ADDR_SIGNAL)) return;
if (!select_signal_address(pernet, msk, &local)) goto subflow;
/* Special case for ID0: set the correct ID */ endp_id = local.addr.id; if (endp_id == msk->mpc_endpoint_id) local.addr.id = 0;
/* If the alloc fails, we are on memory pressure, not worth * continuing, and trying to create subflows. */ if (!mptcp_pm_announced_alloc(msk, &local.addr)) return;
__clear_bit(endp_id, msk->pm.id_avail_bitmap); msk->pm.add_addr_signaled++;
mptcp_pm_announce_addr(msk, &local.addr, false); mptcp_pm_addr_send_ack(msk);
if (local.flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) signal_and_subflow = true; }
subflow: /* No need to try establishing subflows to remote id0 if not allowed */ if (mptcp_pm_add_addr_c_flag_case(msk)) goto exit;
/* check if should create a new subflow */ while (msk->pm.local_addr_used < endp_subflow_max && msk->pm.extra_subflows < limit_extra_subflows) { struct mptcp_addr_info addrs[MPTCP_PM_ADDR_MAX]; bool fullmesh; int i, nr;
if (signal_and_subflow) signal_and_subflow = false; else if (!select_local_address(pernet, msk, &local)) break;
fullmesh = !!(local.flags & MPTCP_PM_ADDR_FLAG_FULLMESH);
__clear_bit(local.addr.id, msk->pm.id_avail_bitmap);
/* Special case for ID0: set the correct ID */ if (local.addr.id == msk->mpc_endpoint_id) local.addr.id = 0; else /* local_addr_used is not decr for ID 0 */ msk->pm.local_addr_used++;
nr = fill_remote_addresses_vec(msk, &local.addr, fullmesh, addrs, ARRAY_SIZE(addrs)); if (nr == 0) continue;
spin_unlock_bh(&msk->pm.lock); for (i = 0; i < nr; i++) __mptcp_subflow_connect(sk, &local, &addrs[i]); spin_lock_bh(&msk->pm.lock); }
exit: /* If an endpoint has both the signal and subflow flags, but it is not * possible to create subflows -- the 'while' loop body above never * executed -- then still mark the endp as used, which is somehow the * case. This avoids issues later when removing the endpoint and calling * __mark_subflow_endp_available(), which expects the increment here. */ if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id) msk->pm.local_addr_used++;
mptcp_pm_nl_check_work_pending(msk);}
static void mptcp_pm_nl_fully_established(struct mptcp_sock *msk){ mptcp_pm_create_subflow_or_signal_addr(msk);}
static void mptcp_pm_nl_subflow_established(struct mptcp_sock *msk){ mptcp_pm_create_subflow_or_signal_addr(msk);}
static unsigned intfill_local_addresses_vec_fullmesh(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals, int locals_size, bool c_flag_case){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); struct sock *sk = (struct sock *)msk; struct mptcp_pm_addr_entry *entry; struct mptcp_pm_local *local; int i = 0;
rcu_read_lock(); list_for_each_entry_rcu(entry, &pernet->endp_list, list) { bool is_id0;
if (!(entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH)) continue;
if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote)) continue;
local = &locals[i]; local->addr = entry->addr; local->flags = entry->flags; local->ifindex = entry->ifindex;
is_id0 = local->addr.id == msk->mpc_endpoint_id;
if (c_flag_case && (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) { __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
if (!is_id0) msk->pm.local_addr_used++; }
/* Special case for ID0: set the correct ID */ if (is_id0) local->addr.id = 0;
msk->pm.extra_subflows++; i++;
if (msk->pm.extra_subflows >= limit_extra_subflows || i == locals_size) break; } rcu_read_unlock();
return i;}
static unsigned intfill_local_laminar_endp(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; struct mptcp_pm_addr_entry *entry; struct mptcp_pm_local *local; int found = 0;
/* Forbid creation of new subflows matching existing ones, possibly * already created by 'subflow' endpoints */ bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
if ((1 << inet_sk_state_load(ssk)) & (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING | TCPF_CLOSE)) continue;
__set_bit(subflow_get_local_id(subflow), unavail_id); }
rcu_read_lock(); list_for_each_entry_rcu(entry, &pernet->endp_list, list) { if (!(entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR)) continue;
if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote)) continue;
if (test_bit(mptcp_endp_get_local_id(msk, &entry->addr), unavail_id)) continue;
local = &locals[0]; local->addr = entry->addr; local->flags = entry->flags; local->ifindex = entry->ifindex;
if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
if (local->addr.id != msk->mpc_endpoint_id) msk->pm.local_addr_used++; }
msk->pm.extra_subflows++; found = 1; break; } rcu_read_unlock();
return found;}
static unsigned intfill_local_addresses_vec_c_flag(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals, int locals_size){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk); struct sock *sk = (struct sock *)msk; struct mptcp_pm_local *local; int i = 0;
while (msk->pm.local_addr_used < endp_subflow_max) { local = &locals[i];
if (!select_local_address(pernet, msk, local)) break;
__clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
if (!mptcp_pm_addr_families_match(sk, &local->addr, remote)) continue;
if (local->addr.id == msk->mpc_endpoint_id) continue;
msk->pm.local_addr_used++; msk->pm.extra_subflows++; i++;
if (msk->pm.extra_subflows >= limit_extra_subflows || i == locals_size) break; }
return i;}
static unsigned intfill_local_address_any(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *local){ struct sock *sk = (struct sock *)msk;
memset(local, 0, sizeof(*local)); local->addr.family =#if IS_ENABLED(CONFIG_MPTCP_IPV6) remote->family == AF_INET6 && ipv6_addr_v4mapped(&remote->addr6) ? AF_INET :#endif remote->family;
if (!mptcp_pm_addr_families_match(sk, &local->addr, remote)) return 0;
msk->pm.extra_subflows++;
return 1;}
/* Fill all the local addresses into the array addrs[], * and return the array size. */static unsigned intfill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals, int locals_size){ bool c_flag_case = remote->id && mptcp_pm_add_addr_c_flag_case(msk);
/* If there is at least one MPTCP endpoint with a fullmesh flag */ if (mptcp_pm_get_endp_fullmesh_max(msk)) return fill_local_addresses_vec_fullmesh(msk, remote, locals, locals_size, c_flag_case);
/* If there is at least one MPTCP endpoint with a laminar flag */ if (mptcp_pm_get_endp_laminar_max(msk)) return fill_local_laminar_endp(msk, remote, locals);
/* Special case: peer sets the C flag, accept one ADD_ADDR if default * limits are used -- accepting no ADD_ADDR -- and use subflow endpoints */ if (c_flag_case) return fill_local_addresses_vec_c_flag(msk, remote, locals, locals_size);
/* No special case: fill in the single 'IPADDRANY' local address */ return fill_local_address_any(msk, remote, &locals[0]);}
static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk){ u8 limit_add_addr_accepted = mptcp_pm_get_limit_add_addr_accepted(msk); u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct mptcp_pm_local locals[MPTCP_PM_ADDR_MAX]; struct sock *sk = (struct sock *)msk; struct mptcp_addr_info remote; bool sf_created = false; int i, nr;
pr_debug("accepted %d:%d remote family %d\n", msk->pm.add_addr_accepted, limit_add_addr_accepted, msk->pm.remote.family);
remote = msk->pm.remote; mptcp_pm_announce_addr(msk, &remote, true); mptcp_pm_addr_send_ack(msk); mptcp_mpc_endpoint_setup(msk);
if (has_subflow_daddr(msk, &remote)) return;
/* pick id 0 port, if none is provided the remote address */ if (!remote.port) remote.port = sk->sk_dport;
/* connect to the specified remote address, using whatever * local address the routing configuration will pick. */ nr = fill_local_addresses_vec(msk, &remote, locals, ARRAY_SIZE(locals)); if (nr == 0) return;
spin_unlock_bh(&msk->pm.lock); for (i = 0; i < nr; i++) if (__mptcp_subflow_connect(sk, &locals[i], &remote) == 0) sf_created = true; spin_lock_bh(&msk->pm.lock);
if (sf_created) { /* add_addr_accepted is not decr for ID 0 */ if (remote.id) msk->pm.add_addr_accepted++; if (msk->pm.add_addr_accepted >= limit_add_addr_accepted || msk->pm.extra_subflows >= limit_extra_subflows) WRITE_ONCE(msk->pm.accept_addr, false); }}
void mptcp_pm_nl_rm_addr(struct mptcp_sock *msk, u8 rm_id){ if (rm_id && !WARN_ON_ONCE(msk->pm.add_addr_accepted == 0)) { u8 limit_add_addr_accepted = mptcp_pm_get_limit_add_addr_accepted(msk);
/* Note: if the subflow has been closed before, this * add_addr_accepted counter will not be decremented. */ if (--msk->pm.add_addr_accepted < limit_add_addr_accepted) WRITE_ONCE(msk->pm.accept_addr, true); }}
static bool address_use_port(struct mptcp_pm_addr_entry *entry){ return (entry->flags & (MPTCP_PM_ADDR_FLAG_SIGNAL | MPTCP_PM_ADDR_FLAG_SUBFLOW)) == MPTCP_PM_ADDR_FLAG_SIGNAL;}
/* caller must ensure the RCU grace period is already elapsed */static void __mptcp_pm_release_addr_entry(struct mptcp_pm_addr_entry *entry){ if (entry->lsk) sock_release(entry->lsk); kfree(entry);}
static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, struct mptcp_pm_addr_entry *entry, bool replace){ struct mptcp_pm_addr_entry *cur, *del_entry = NULL; int ret = -EINVAL; u8 addr_max;
spin_lock_bh(&pernet->lock); /* to keep the code simple, don't do IDR-like allocation for address ID, * just bail when we exceed limits */ if (pernet->next_id == MPTCP_PM_MAX_ADDR_ID) pernet->next_id = 1; if (pernet->endpoints == MPTCP_PM_MAX_ADDR_ID) { ret = -ERANGE; goto out; } if (test_bit(entry->addr.id, pernet->id_bitmap)) { ret = -EBUSY; goto out; }
/* do not insert duplicate address, differentiate on port only * singled addresses */ if (!address_use_port(entry)) entry->addr.port = 0; list_for_each_entry(cur, &pernet->endp_list, list) { if (mptcp_addresses_equal(&cur->addr, &entry->addr, cur->addr.port || entry->addr.port)) { /* allow replacing the exiting endpoint only if such * endpoint is an implicit one and the user-space * did not provide an endpoint id */ if (!(cur->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)) { ret = -EEXIST; goto out; } if (entry->addr.id) goto out;
/* allow callers that only need to look up the local * addr's id to skip replacement. This allows them to * avoid calling synchronize_rcu in the packet recv * path. */ if (!replace) { kfree(entry); ret = cur->addr.id; goto out; }
pernet->endpoints--; entry->addr.id = cur->addr.id; list_del_rcu(&cur->list); del_entry = cur; break; } }
if (!entry->addr.id) {find_next: entry->addr.id = find_next_zero_bit(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1, pernet->next_id); if (!entry->addr.id && pernet->next_id != 1) { pernet->next_id = 1; goto find_next; } }
if (!entry->addr.id) goto out;
__set_bit(entry->addr.id, pernet->id_bitmap); if (entry->addr.id > pernet->next_id) pernet->next_id = entry->addr.id;
if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) { addr_max = pernet->endp_signal_max; WRITE_ONCE(pernet->endp_signal_max, addr_max + 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { addr_max = pernet->endp_subflow_max; WRITE_ONCE(pernet->endp_subflow_max, addr_max + 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) { addr_max = pernet->endp_laminar_max; WRITE_ONCE(pernet->endp_laminar_max, addr_max + 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) { addr_max = pernet->endp_fullmesh_max; WRITE_ONCE(pernet->endp_fullmesh_max, addr_max + 1); }
pernet->endpoints++; if (!entry->addr.port) list_add_tail_rcu(&entry->list, &pernet->endp_list); else list_add_rcu(&entry->list, &pernet->endp_list); ret = entry->addr.id;
out: spin_unlock_bh(&pernet->lock);
/* just replaced an existing entry, free it */ if (del_entry) { synchronize_rcu(); __mptcp_pm_release_addr_entry(del_entry); } return ret;}
static struct lock_class_key mptcp_slock_keys[2];static struct lock_class_key mptcp_keys[2];
static int mptcp_pm_nl_create_listen_socket(struct sock *sk, struct mptcp_pm_addr_entry *entry){ bool is_ipv6 = entry->addr.family == AF_INET6; int addrlen = sizeof(struct sockaddr_in); struct sockaddr_storage addr; struct sock *newsk, *ssk; int backlog = 1024; int err;
err = sock_create_kern(sock_net(sk), entry->addr.family, SOCK_STREAM, IPPROTO_MPTCP, &entry->lsk); if (err) return err;
newsk = entry->lsk->sk; if (!newsk) return -EINVAL;
/* The subflow socket lock is acquired in a nested to the msk one * in several places, even by the TCP stack, and this msk is a kernel * socket: lockdep complains. Instead of propagating the _nested * modifiers in several places, re-init the lock class for the msk * socket to an mptcp specific one. */ sock_lock_init_class_and_name(newsk, is_ipv6 ? "mlock-AF_INET6" : "mlock-AF_INET", &mptcp_slock_keys[is_ipv6], is_ipv6 ? "msk_lock-AF_INET6" : "msk_lock-AF_INET", &mptcp_keys[is_ipv6]);
lock_sock(newsk); ssk = __mptcp_nmpc_sk(mptcp_sk(newsk)); release_sock(newsk); if (IS_ERR(ssk)) return PTR_ERR(ssk);
mptcp_info2sockaddr(&entry->addr, &addr, entry->addr.family);#if IS_ENABLED(CONFIG_MPTCP_IPV6) if (entry->addr.family == AF_INET6) addrlen = sizeof(struct sockaddr_in6);#endif if (ssk->sk_family == AF_INET) err = inet_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);#if IS_ENABLED(CONFIG_MPTCP_IPV6) else if (ssk->sk_family == AF_INET6) err = inet6_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);#endif if (err) return err;
/* We don't use mptcp_set_state() here because it needs to be called * under the msk socket lock. For the moment, that will not bring * anything more than only calling inet_sk_state_store(), because the * old status is known (TCP_CLOSE). */ inet_sk_state_store(newsk, TCP_LISTEN); lock_sock(ssk); WRITE_ONCE(mptcp_subflow_ctx(ssk)->pm_listener, true); err = __inet_listen_sk(ssk, backlog); if (!err) mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED); release_sock(ssk); return err;}
int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_pm_addr_entry *skc){ struct mptcp_pm_addr_entry *entry; struct pm_nl_pernet *pernet; int ret;
pernet = pm_nl_get_pernet_from_msk(msk);
rcu_read_lock(); entry = __lookup_addr(pernet, &skc->addr); ret = entry ? entry->addr.id : -1; rcu_read_unlock(); if (ret >= 0) return ret;
/* address not found, add to local list */ entry = kmemdup(skc, sizeof(*skc), GFP_ATOMIC); if (!entry) return -ENOMEM;
entry->addr.port = 0; ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, false); if (ret < 0) kfree(entry);
return ret;}
bool mptcp_pm_nl_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); struct mptcp_pm_addr_entry *entry; bool backup;
rcu_read_lock(); entry = __lookup_addr(pernet, skc); backup = entry && !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP); rcu_read_unlock();
return backup;}
static int mptcp_nl_add_subflow_or_signal_addr(struct net *net, struct mptcp_addr_info *addr){ struct mptcp_sock *msk; long s_slot = 0, s_num = 0;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; struct mptcp_addr_info mpc_addr;
if (!READ_ONCE(msk->fully_established) || mptcp_pm_is_userspace(msk)) goto next;
/* if the endp linked to the init sf is re-added with a != ID */ mptcp_local_address((struct sock_common *)msk, &mpc_addr);
lock_sock(sk); spin_lock_bh(&msk->pm.lock); if (mptcp_addresses_equal(addr, &mpc_addr, addr->port)) msk->mpc_endpoint_id = addr->id; mptcp_pm_create_subflow_or_signal_addr(msk); spin_unlock_bh(&msk->pm.lock); release_sock(sk);
next: sock_put(sk); cond_resched(); }
return 0;}
/* Add an MPTCP endpoint */int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct mptcp_pm_addr_entry addr, *entry; struct nlattr *attr; int ret;
if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR)) return -EINVAL;
attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR]; ret = mptcp_pm_parse_entry(attr, info, true, &addr); if (ret < 0) return ret;
if (addr.addr.port && !address_use_port(&addr)) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "flags must have signal and not subflow when using port"); return -EINVAL; }
if (addr.flags & MPTCP_PM_ADDR_FLAG_SIGNAL && addr.flags & MPTCP_PM_ADDR_FLAG_FULLMESH) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "flags mustn't have both signal and fullmesh"); return -EINVAL; }
if (addr.flags & MPTCP_PM_ADDR_FLAG_IMPLICIT) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "can't create IMPLICIT endpoint"); return -EINVAL; }
entry = kmemdup(&addr, sizeof(addr), GFP_KERNEL_ACCOUNT); if (!entry) { GENL_SET_ERR_MSG(info, "can't allocate addr"); return -ENOMEM; }
if (entry->addr.port) { ret = mptcp_pm_nl_create_listen_socket(skb->sk, entry); if (ret) { GENL_SET_ERR_MSG_FMT(info, "create listen socket error: %d", ret); goto out_free; } } ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true); if (ret < 0) { GENL_SET_ERR_MSG_FMT(info, "too many addresses or duplicate one: %d", ret); goto out_free; }
mptcp_nl_add_subflow_or_signal_addr(sock_net(skb->sk), &entry->addr); return 0;
out_free: __mptcp_pm_release_addr_entry(entry); return ret;}
static void mptcp_pm_remove_announced(struct mptcp_sock *msk, const struct mptcp_addr_info *addr, bool force){ struct mptcp_rm_list list = { .nr = 0 }; bool announced;
list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
announced = mptcp_pm_announced_remove(msk, addr); if (announced || force) { spin_lock_bh(&msk->pm.lock); if (announced) msk->pm.add_addr_signaled--; mptcp_pm_remove_addr(msk, &list); spin_unlock_bh(&msk->pm.lock); }}
static void __mark_subflow_endp_available(struct mptcp_sock *msk, u8 id){ /* If it was marked as used, and not ID 0, decrement local_addr_used */ if (!__test_and_set_bit(id ? : msk->mpc_endpoint_id, msk->pm.id_avail_bitmap) && id && !WARN_ON_ONCE(msk->pm.local_addr_used == 0)) msk->pm.local_addr_used--;}
static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net, const struct mptcp_pm_addr_entry *entry){ const struct mptcp_addr_info *addr = &entry->addr; struct mptcp_rm_list list = { .nr = 1 }; long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
pr_debug("remove_id=%d\n", addr->id);
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; bool remove_subflow;
if (mptcp_pm_is_userspace(msk)) goto next;
lock_sock(sk); remove_subflow = mptcp_pm_has_subflow_saddr(msk, addr); mptcp_pm_remove_announced(msk, addr, remove_subflow && !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT));
list.ids[0] = mptcp_endp_get_local_id(msk, addr);
spin_lock_bh(&msk->pm.lock); if (remove_subflow) mptcp_pm_rm_subflow(msk, &list); if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) __mark_subflow_endp_available(msk, list.ids[0]); else /* mark endp ID as available, e.g. Signal or MPC endp */ __set_bit(addr->id, msk->pm.id_avail_bitmap); spin_unlock_bh(&msk->pm.lock);
if (msk->mpc_endpoint_id == entry->addr.id) msk->mpc_endpoint_id = 0; release_sock(sk);
next: sock_put(sk); cond_resched(); }
return 0;}
static int mptcp_nl_remove_id_zero_address(struct net *net, struct mptcp_addr_info *addr){ struct mptcp_rm_list list = { .nr = 0 }; long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
list.ids[list.nr++] = 0;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; struct mptcp_addr_info msk_local;
if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk)) goto next;
mptcp_local_address((struct sock_common *)msk, &msk_local); if (!mptcp_addresses_equal(&msk_local, addr, addr->port)) goto next;
lock_sock(sk); spin_lock_bh(&msk->pm.lock); mptcp_pm_remove_addr(msk, &list); mptcp_pm_rm_subflow(msk, &list); __mark_subflow_endp_available(msk, 0); spin_unlock_bh(&msk->pm.lock); release_sock(sk);
next: sock_put(sk); cond_resched(); }
return 0;}
/* Remove an MPTCP endpoint */int mptcp_pm_nl_del_addr_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct mptcp_pm_addr_entry addr, *entry; struct nlattr *attr; u8 addr_max; int ret;
if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR)) return -EINVAL;
attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR]; ret = mptcp_pm_parse_entry(attr, info, false, &addr); if (ret < 0) return ret;
/* the zero id address is special: the first address used by the msk * always gets such an id, so different subflows can have different zero * id addresses. Additionally zero id is not accounted for in id_bitmap. * Let's use an 'mptcp_rm_list' instead of the common remove code. */ if (addr.addr.id == 0) return mptcp_nl_remove_id_zero_address(sock_net(skb->sk), &addr.addr);
spin_lock_bh(&pernet->lock); entry = __lookup_addr_by_id(pernet, addr.addr.id); if (!entry) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found"); spin_unlock_bh(&pernet->lock); return -EINVAL; } if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) { addr_max = pernet->endp_signal_max; WRITE_ONCE(pernet->endp_signal_max, addr_max - 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { addr_max = pernet->endp_subflow_max; WRITE_ONCE(pernet->endp_subflow_max, addr_max - 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) { addr_max = pernet->endp_laminar_max; WRITE_ONCE(pernet->endp_laminar_max, addr_max - 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) { addr_max = pernet->endp_fullmesh_max; WRITE_ONCE(pernet->endp_fullmesh_max, addr_max - 1); }
pernet->endpoints--; list_del_rcu(&entry->list); __clear_bit(entry->addr.id, pernet->id_bitmap); spin_unlock_bh(&pernet->lock);
mptcp_nl_remove_subflow_and_signal_addr(sock_net(skb->sk), entry); synchronize_rcu(); __mptcp_pm_release_addr_entry(entry);
return ret;}
static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk, struct list_head *rm_list, struct mptcp_pm_addr_entry *entry){ struct mptcp_rm_list alist, slist; bool more;
again: alist.nr = 0; slist.nr = 0; more = false;
entry = list_prepare_entry(entry, rm_list, list); list_for_each_entry_continue(entry, rm_list, list) { if (mptcp_pm_has_subflow_saddr(msk, &entry->addr)) slist.ids[slist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
if (mptcp_pm_announced_remove(msk, &entry->addr)) alist.ids[alist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
if (slist.nr == MPTCP_RM_IDS_MAX || alist.nr == MPTCP_RM_IDS_MAX) { more = !list_is_last(&entry->list, rm_list); break; } }
spin_lock_bh(&msk->pm.lock); if (alist.nr) { msk->pm.add_addr_signaled -= alist.nr; mptcp_pm_remove_addr(msk, &alist); } if (slist.nr) mptcp_pm_rm_subflow(msk, &slist); /* Reset counters: maybe some subflows have been removed before */ if (!more) { bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); msk->pm.local_addr_used = 0; } spin_unlock_bh(&msk->pm.lock);
if (more) goto again;}
static void mptcp_nl_flush_addrs_list(struct net *net, struct list_head *rm_list){ long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
if (list_empty(rm_list)) return;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk;
if (!mptcp_pm_is_userspace(msk)) { lock_sock(sk); mptcp_pm_flush_addrs_and_subflows(msk, rm_list, NULL); release_sock(sk); }
sock_put(sk); cond_resched(); }}
/* caller must ensure the RCU grace period is already elapsed */static void __flush_addrs(struct list_head *list){ while (!list_empty(list)) { struct mptcp_pm_addr_entry *cur;
cur = list_entry(list->next, struct mptcp_pm_addr_entry, list); list_del_rcu(&cur->list); __mptcp_pm_release_addr_entry(cur); }}
static void __reset_counters(struct pm_nl_pernet *pernet){ WRITE_ONCE(pernet->endp_signal_max, 0); WRITE_ONCE(pernet->endp_subflow_max, 0); WRITE_ONCE(pernet->endp_laminar_max, 0); WRITE_ONCE(pernet->endp_fullmesh_max, 0); pernet->endpoints = 0;}
int mptcp_pm_nl_flush_addrs_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct list_head free_list;
spin_lock_bh(&pernet->lock); free_list = pernet->endp_list; INIT_LIST_HEAD_RCU(&pernet->endp_list); __reset_counters(pernet); pernet->next_id = 1; bitmap_zero(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); spin_unlock_bh(&pernet->lock);
if (free_list.next == &pernet->endp_list) return 0;
synchronize_rcu();
/* Adjust the pointers to free_list instead of pernet->endp_list */ free_list.prev->next = &free_list; free_list.next->prev = &free_list;
mptcp_nl_flush_addrs_list(sock_net(skb->sk), &free_list); __flush_addrs(&free_list); return 0;}
int mptcp_pm_nl_get_addr(u8 id, struct mptcp_pm_addr_entry *addr, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct mptcp_pm_addr_entry *entry; int ret = -EINVAL;
rcu_read_lock(); entry = __lookup_addr_by_id(pernet, id); if (entry) { *addr = *entry; ret = 0; } rcu_read_unlock();
return ret;}
int mptcp_pm_nl_dump_addr(struct sk_buff *msg, struct netlink_callback *cb){ struct net *net = sock_net(msg->sk); struct mptcp_pm_addr_entry *entry; struct pm_nl_pernet *pernet; int id = cb->args[0]; int i;
pernet = pm_nl_get_pernet(net);
rcu_read_lock(); for (i = id; i < MPTCP_PM_MAX_ADDR_ID + 1; i++) { if (test_bit(i, pernet->id_bitmap)) { entry = __lookup_addr_by_id(pernet, i); if (!entry) break;
if (entry->addr.id <= id) continue;
if (mptcp_pm_genl_fill_addr(msg, cb, entry) < 0) break;
id = entry->addr.id; } } rcu_read_unlock();
cb->args[0] = id; return msg->len;}
static int parse_limit(struct genl_info *info, int id, unsigned int *limit){ struct nlattr *attr = info->attrs[id];
if (!attr) return 0;
*limit = nla_get_u32(attr); if (*limit > MPTCP_PM_SUBFLOWS_MAX) { NL_SET_ERR_MSG_ATTR_FMT(info->extack, attr, "limit greater than maximum (%u)", MPTCP_PM_SUBFLOWS_MAX); return -EINVAL; } return 0;}
int mptcp_pm_nl_set_limits_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); unsigned int rcv_addrs, subflows; int ret;
spin_lock_bh(&pernet->lock); rcv_addrs = pernet->limit_add_addr_accepted; ret = parse_limit(info, MPTCP_PM_ATTR_RCV_ADD_ADDRS, &rcv_addrs); if (ret) goto unlock;
subflows = pernet->limit_extra_subflows; ret = parse_limit(info, MPTCP_PM_ATTR_SUBFLOWS, &subflows); if (ret) goto unlock;
WRITE_ONCE(pernet->limit_add_addr_accepted, rcv_addrs); WRITE_ONCE(pernet->limit_extra_subflows, subflows);
unlock: spin_unlock_bh(&pernet->lock); return ret;}
int mptcp_pm_nl_get_limits_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct sk_buff *msg; void *reply;
msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!msg) return -ENOMEM;
reply = genlmsg_put_reply(msg, info, &mptcp_genl_family, 0, MPTCP_PM_CMD_GET_LIMITS); if (!reply) goto fail;
if (nla_put_u32(msg, MPTCP_PM_ATTR_RCV_ADD_ADDRS, READ_ONCE(pernet->limit_add_addr_accepted))) goto fail;
if (nla_put_u32(msg, MPTCP_PM_ATTR_SUBFLOWS, READ_ONCE(pernet->limit_extra_subflows))) goto fail;
genlmsg_end(msg, reply); return genlmsg_reply(msg, info);
fail: GENL_SET_ERR_MSG(info, "not enough space in Netlink message"); nlmsg_free(msg); return -EMSGSIZE;}
static void mptcp_pm_nl_fullmesh(struct mptcp_sock *msk, struct mptcp_addr_info *addr){ struct mptcp_rm_list list = { .nr = 0 };
list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
spin_lock_bh(&msk->pm.lock); mptcp_pm_rm_subflow(msk, &list); __mark_subflow_endp_available(msk, list.ids[0]); mptcp_pm_create_subflow_or_signal_addr(msk); spin_unlock_bh(&msk->pm.lock);}
static void mptcp_pm_nl_set_flags_all(struct net *net, struct mptcp_pm_addr_entry *local, u8 changed){ u8 is_subflow = !!(local->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW); u8 bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP); long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
if (changed == MPTCP_PM_ADDR_FLAG_FULLMESH && !is_subflow) return;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk;
if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk)) goto next;
lock_sock(sk); if (changed & MPTCP_PM_ADDR_FLAG_BACKUP) mptcp_pm_mp_prio_send_ack(msk, &local->addr, NULL, bkup); /* Subflows will only be recreated if the SUBFLOW flag is set */ if (is_subflow && (changed & MPTCP_PM_ADDR_FLAG_FULLMESH)) mptcp_pm_nl_fullmesh(msk, &local->addr); release_sock(sk);
next: sock_put(sk); cond_resched(); }}
int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local, struct genl_info *info){ struct nlattr *attr = info->attrs[MPTCP_PM_ATTR_ADDR]; u8 changed, mask = MPTCP_PM_ADDR_FLAG_BACKUP | MPTCP_PM_ADDR_FLAG_FULLMESH; struct net *net = genl_info_net(info); struct mptcp_pm_addr_entry *entry; struct pm_nl_pernet *pernet; u8 lookup_by_id = 0;
pernet = pm_nl_get_pernet(net);
if (local->addr.family == AF_UNSPEC) { lookup_by_id = 1; if (!local->addr.id) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "missing address ID"); return -EOPNOTSUPP; } }
spin_lock_bh(&pernet->lock); entry = lookup_by_id ? __lookup_addr_by_id(pernet, local->addr.id) : __lookup_addr(pernet, &local->addr); if (!entry) { spin_unlock_bh(&pernet->lock); NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found"); return -EINVAL; } if ((local->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) && (entry->flags & (MPTCP_PM_ADDR_FLAG_SIGNAL | MPTCP_PM_ADDR_FLAG_IMPLICIT))) { spin_unlock_bh(&pernet->lock); NL_SET_ERR_MSG_ATTR(info->extack, attr, "invalid addr flags"); return -EINVAL; }
changed = (local->flags ^ entry->flags) & mask; entry->flags = (entry->flags & ~mask) | (local->flags & mask); *local = *entry;
if (changed & MPTCP_PM_ADDR_FLAG_FULLMESH) { u8 addr_max = pernet->endp_fullmesh_max;
if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) addr_max++; else addr_max--;
WRITE_ONCE(pernet->endp_fullmesh_max, addr_max); }
spin_unlock_bh(&pernet->lock);
mptcp_pm_nl_set_flags_all(net, local, changed); return 0;}
bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
if (msk->pm.extra_subflows == mptcp_pm_get_limit_extra_subflows(msk) || (find_next_and_bit(pernet->id_bitmap, msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1, 0) == MPTCP_PM_MAX_ADDR_ID + 1)) { WRITE_ONCE(msk->pm.work_pending, false); return false; } return true;}
/* Called under PM lock */void __mptcp_pm_kernel_worker(struct mptcp_sock *msk){ struct mptcp_pm_data *pm = &msk->pm;
if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) { pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED); mptcp_pm_nl_add_addr_received(msk); } if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) { pm->status &= ~BIT(MPTCP_PM_ESTABLISHED); mptcp_pm_nl_fully_established(msk); } if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) { pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED); mptcp_pm_nl_subflow_established(msk); }}
static int __net_init pm_nl_init_net(struct net *net){ struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
INIT_LIST_HEAD_RCU(&pernet->endp_list);
/* Cit. 2 subflows ought to be enough for anybody. */ pernet->limit_extra_subflows = 2; pernet->next_id = 1; spin_lock_init(&pernet->lock);
/* No need to initialize other pernet fields, the struct is zeroed at * allocation time. */
return 0;}
static void __net_exit pm_nl_exit_net(struct list_head *net_list){ struct net *net;
list_for_each_entry(net, net_list, exit_list) { struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
/* net is removed from namespace list, can't race with * other modifiers, also netns core already waited for a * RCU grace period. */ __flush_addrs(&pernet->endp_list); }}
static struct pernet_operations mptcp_pm_pernet_ops = { .init = pm_nl_init_net, .exit_batch = pm_nl_exit_net, .id = &pm_nl_pernet_id, .size = sizeof(struct pm_nl_pernet),};
struct mptcp_pm_ops mptcp_pm_kernel = { .name = "kernel", .owner = THIS_MODULE,};
void __init mptcp_pm_kernel_register(void){ if (register_pernet_subsys(&mptcp_pm_pernet_ops) < 0) panic("Failed to register MPTCP PM pernet subsystem.\n");
mptcp_pm_register(&mptcp_pm_kernel);}