Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664// SPDX-License-Identifier: GPL-2.0/* Multipath TCP * * Copyright (c) 2025, Matthieu Baerts. */
#define pr_fmt(fmt) "MPTCP: " fmt
#include <net/netns/generic.h>
#include "protocol.h"#include "mib.h"#include "mptcp_pm_gen.h"
static int pm_nl_pernet_id;
struct pm_nl_pernet { /* protects pernet updates */ spinlock_t lock; struct list_head endp_list; u8 endpoints; u8 endp_signal_max; u8 endp_subflow_max; u8 endp_laminar_max; u8 endp_fullmesh_max; u8 limit_add_addr_accepted; u8 limit_extra_subflows; u8 next_id; DECLARE_BITMAP(id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);};
#define MPTCP_PM_ADDR_MAX 8#define MPTCP_PM_SUBFLOWS_MAX 64
static struct pm_nl_pernet *pm_nl_get_pernet(const struct net *net){ return net_generic(net, pm_nl_pernet_id);}
static struct pm_nl_pernet *pm_nl_get_pernet_from_msk(const struct mptcp_sock *msk){ return pm_nl_get_pernet(sock_net((struct sock *)msk));}
static struct pm_nl_pernet *genl_info_pm_nl(struct genl_info *info){ return pm_nl_get_pernet(genl_info_net(info));}
u8 mptcp_pm_get_endp_signal_max(const struct mptcp_sock *msk){ const struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_signal_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_signal_max);
u8 mptcp_pm_get_endp_subflow_max(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_subflow_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_subflow_max);
u8 mptcp_pm_get_endp_laminar_max(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_laminar_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_laminar_max);
u8 mptcp_pm_get_endp_fullmesh_max(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->endp_fullmesh_max);}EXPORT_SYMBOL_GPL(mptcp_pm_get_endp_fullmesh_max);
u8 mptcp_pm_get_limit_add_addr_accepted(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->limit_add_addr_accepted);}EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_add_addr_accepted);
u8 mptcp_pm_get_limit_extra_subflows(const struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
return READ_ONCE(pernet->limit_extra_subflows);}EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_extra_subflows);
static bool has_subflow_daddr(const struct mptcp_sock *msk, const struct mptcp_addr_info *daddr){ struct mptcp_subflow_context *subflow; struct mptcp_addr_info cur;
mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
if (!((1 << inet_sk_state_load(ssk)) & (TCPF_ESTABLISHED | TCPF_SYN_SENT | TCPF_SYN_RECV))) continue;
mptcp_remote_address((struct sock_common *)ssk, &cur); if (mptcp_addresses_equal(&cur, daddr, daddr->port)) return true; }
return false;}
static boolselect_local_address(const struct pm_nl_pernet *pernet, const struct mptcp_sock *msk, struct mptcp_pm_local *new_local){ struct mptcp_pm_addr_entry *entry; bool found = false;
msk_owned_by_me(msk);
rcu_read_lock(); list_for_each_entry_rcu(entry, &pernet->endp_list, list) { if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) continue;
if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap)) continue;
new_local->addr = entry->addr; new_local->flags = entry->flags; new_local->ifindex = entry->ifindex; found = true; break; } rcu_read_unlock();
return found;}
static boolselect_signal_address(struct pm_nl_pernet *pernet, const struct mptcp_sock *msk, struct mptcp_pm_local *new_local){ struct mptcp_pm_addr_entry *entry; bool found = false;
rcu_read_lock(); /* do not keep any additional per socket state, just signal * the address list in order. * Note: removal from the local address list during the msk life-cycle * can lead to additional addresses not being announced. */ list_for_each_entry_rcu(entry, &pernet->endp_list, list) { if (!test_bit(entry->addr.id, msk->pm.id_avail_bitmap)) continue;
if (!(entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL)) continue;
new_local->addr = entry->addr; new_local->flags = entry->flags; new_local->ifindex = entry->ifindex; found = true; break; } rcu_read_unlock();
return found;}
static unsigned intfill_remote_addr(struct mptcp_sock *msk, struct mptcp_addr_info *local, struct mptcp_addr_info *addrs){ bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0); struct mptcp_addr_info remote = { 0 }; struct sock *sk = (struct sock *)msk;
if (deny_id0) return 0;
mptcp_remote_address((struct sock_common *)sk, &remote);
if (!mptcp_pm_addr_families_match(sk, local, &remote)) return 0;
msk->pm.extra_subflows++; *addrs = remote;
return 1;}
static unsigned intfill_remote_addresses_fullmesh(struct mptcp_sock *msk, struct mptcp_addr_info *local, struct mptcp_addr_info *addrs, int addrs_size){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0); DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); struct sock *sk = (struct sock *)msk, *ssk; struct mptcp_subflow_context *subflow; int i = 0;
/* Forbid creation of new subflows matching existing ones, possibly * already created by incoming ADD_ADDR */ bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); mptcp_for_each_subflow(msk, subflow) if (READ_ONCE(subflow->local_id) == local->id) __set_bit(subflow->remote_id, unavail_id);
mptcp_for_each_subflow(msk, subflow) { ssk = mptcp_subflow_tcp_sock(subflow); mptcp_remote_address((struct sock_common *)ssk, &addrs[i]); addrs[i].id = READ_ONCE(subflow->remote_id); if (deny_id0 && !addrs[i].id) continue;
if (test_bit(addrs[i].id, unavail_id)) continue;
if (!mptcp_pm_addr_families_match(sk, local, &addrs[i])) continue;
/* forbid creating multiple address towards this id */ __set_bit(addrs[i].id, unavail_id); msk->pm.extra_subflows++; i++;
if (msk->pm.extra_subflows >= limit_extra_subflows || i == addrs_size) break; }
return i;}
/* Fill all the remote addresses into the array addrs[], * and return the array size. */static unsigned intfill_remote_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *local, bool fullmesh, struct mptcp_addr_info *addrs, int addrs_size){ /* Non-fullmesh: fill in the single entry corresponding to the primary * MPC subflow remote address, and return 1, corresponding to 1 entry. */ if (!fullmesh) return fill_remote_addr(msk, local, addrs);
/* Fullmesh endpoint: fill all possible remote addresses */ return fill_remote_addresses_fullmesh(msk, local, addrs, addrs_size);}
static struct mptcp_pm_addr_entry *__lookup_addr_by_id(struct pm_nl_pernet *pernet, unsigned int id){ struct mptcp_pm_addr_entry *entry;
list_for_each_entry_rcu(entry, &pernet->endp_list, list, lockdep_is_held(&pernet->lock)) { if (entry->addr.id == id) return entry; } return NULL;}
static struct mptcp_pm_addr_entry *__lookup_addr(struct pm_nl_pernet *pernet, const struct mptcp_addr_info *info){ struct mptcp_pm_addr_entry *entry;
list_for_each_entry_rcu(entry, &pernet->endp_list, list, lockdep_is_held(&pernet->lock)) { if (mptcp_addresses_equal(&entry->addr, info, entry->addr.port)) return entry; } return NULL;}
static u8 mptcp_endp_get_local_id(struct mptcp_sock *msk, const struct mptcp_addr_info *addr){ return msk->mpc_endpoint_id == addr->id ? 0 : addr->id;}
/* Set mpc_endpoint_id, and send MP_PRIO for ID0 if needed */static void mptcp_mpc_endpoint_setup(struct mptcp_sock *msk){ struct mptcp_subflow_context *subflow; struct mptcp_pm_addr_entry *entry; struct mptcp_addr_info mpc_addr; struct pm_nl_pernet *pernet; bool backup = false;
/* do lazy endpoint usage accounting for the MPC subflows */ if (likely(msk->pm.status & BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED)) || !msk->first) return;
subflow = mptcp_subflow_ctx(msk->first); pernet = pm_nl_get_pernet_from_msk(msk);
mptcp_local_address((struct sock_common *)msk->first, &mpc_addr); rcu_read_lock(); entry = __lookup_addr(pernet, &mpc_addr); if (entry) { __clear_bit(entry->addr.id, msk->pm.id_avail_bitmap); msk->mpc_endpoint_id = entry->addr.id; backup = !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP); } rcu_read_unlock();
/* Send MP_PRIO */ if (backup) mptcp_pm_send_ack(msk, subflow, true, backup);
msk->pm.status |= BIT(MPTCP_PM_MPC_ENDPOINT_ACCOUNTED);}
static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk); u8 endp_signal_max = mptcp_pm_get_endp_signal_max(msk); struct sock *sk = (struct sock *)msk; bool signal_and_subflow = false; struct mptcp_pm_local local;
mptcp_mpc_endpoint_setup(msk); if (!mptcp_is_fully_established(sk)) return;
pr_debug("local %d:%d signal %d:%d subflows %d:%d\n", msk->pm.local_addr_used, endp_subflow_max, msk->pm.add_addr_signaled, endp_signal_max, msk->pm.extra_subflows, limit_extra_subflows);
/* check first for announce */ if (msk->pm.add_addr_signaled < endp_signal_max) { u8 endp_id;
/* due to racing events on both ends we can reach here while * previous add address is still running: if we invoke now * mptcp_pm_announce_addr(), that will fail and the * corresponding id will be marked as used. * Instead let the PM machinery reschedule us when the * current address announce will be completed. */ if (msk->pm.addr_signal & BIT(MPTCP_ADD_ADDR_SIGNAL)) return;
if (!select_signal_address(pernet, msk, &local)) goto subflow;
/* Special case for ID0: set the correct ID */ endp_id = local.addr.id; if (endp_id == msk->mpc_endpoint_id) local.addr.id = 0;
/* If the alloc fails, we are on memory pressure, not worth * continuing, and trying to create subflows. */ if (!mptcp_pm_announced_alloc(msk, &local.addr)) return;
__clear_bit(endp_id, msk->pm.id_avail_bitmap); msk->pm.add_addr_signaled++;
mptcp_pm_announce_addr(msk, &local.addr, false); mptcp_pm_addr_send_ack(msk);
if (local.flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) signal_and_subflow = true; }
subflow: /* No need to try establishing subflows to remote id0 if not allowed */ if (mptcp_pm_add_addr_c_flag_case(msk)) goto exit;
/* check if should create a new subflow */ while (msk->pm.local_addr_used < endp_subflow_max && msk->pm.extra_subflows < limit_extra_subflows) { struct mptcp_addr_info addrs[MPTCP_PM_ADDR_MAX]; bool fullmesh; int i, nr;
if (signal_and_subflow) signal_and_subflow = false; else if (!select_local_address(pernet, msk, &local)) break;
fullmesh = !!(local.flags & MPTCP_PM_ADDR_FLAG_FULLMESH);
__clear_bit(local.addr.id, msk->pm.id_avail_bitmap);
/* Special case for ID0: set the correct ID */ if (local.addr.id == msk->mpc_endpoint_id) local.addr.id = 0; else /* local_addr_used is not decr for ID 0 */ msk->pm.local_addr_used++;
nr = fill_remote_addresses_vec(msk, &local.addr, fullmesh, addrs, ARRAY_SIZE(addrs)); if (nr == 0) continue;
spin_unlock_bh(&msk->pm.lock); for (i = 0; i < nr; i++) __mptcp_subflow_connect(sk, &local, &addrs[i]); spin_lock_bh(&msk->pm.lock); }
exit: /* If an endpoint has both the signal and subflow flags, but it is not * possible to create subflows -- the 'while' loop body above never * executed -- then still mark the endp as used, which is somehow the * case. This avoids issues later when removing the endpoint and calling * __mark_subflow_endp_available(), which expects the increment here. */ if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id) msk->pm.local_addr_used++;
mptcp_pm_nl_check_work_pending(msk);}
static void mptcp_pm_nl_fully_established(struct mptcp_sock *msk){ mptcp_pm_create_subflow_or_signal_addr(msk);}
static void mptcp_pm_nl_subflow_established(struct mptcp_sock *msk){ mptcp_pm_create_subflow_or_signal_addr(msk);}
static unsigned intfill_local_addresses_vec_fullmesh(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals, int locals_size, bool c_flag_case){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); struct sock *sk = (struct sock *)msk; struct mptcp_pm_addr_entry *entry; struct mptcp_pm_local *local; int i = 0;
rcu_read_lock(); list_for_each_entry_rcu(entry, &pernet->endp_list, list) { bool is_id0;
if (!(entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH)) continue;
if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote)) continue;
local = &locals[i]; local->addr = entry->addr; local->flags = entry->flags; local->ifindex = entry->ifindex;
is_id0 = local->addr.id == msk->mpc_endpoint_id;
if (c_flag_case && (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW)) { __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
if (!is_id0) msk->pm.local_addr_used++; }
/* Special case for ID0: set the correct ID */ if (is_id0) local->addr.id = 0;
msk->pm.extra_subflows++; i++;
if (msk->pm.extra_subflows >= limit_extra_subflows || i == locals_size) break; } rcu_read_unlock();
return i;}
static unsigned intfill_local_laminar_endp(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); DECLARE_BITMAP(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; struct mptcp_pm_addr_entry *entry; struct mptcp_pm_local *local; int found = 0;
/* Forbid creation of new subflows matching existing ones, possibly * already created by 'subflow' endpoints */ bitmap_zero(unavail_id, MPTCP_PM_MAX_ADDR_ID + 1); mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
if ((1 << inet_sk_state_load(ssk)) & (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING | TCPF_CLOSE)) continue;
__set_bit(subflow_get_local_id(subflow), unavail_id); }
rcu_read_lock(); list_for_each_entry_rcu(entry, &pernet->endp_list, list) { if (!(entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR)) continue;
if (!mptcp_pm_addr_families_match(sk, &entry->addr, remote)) continue;
if (test_bit(mptcp_endp_get_local_id(msk, &entry->addr), unavail_id)) continue;
local = &locals[0]; local->addr = entry->addr; local->flags = entry->flags; local->ifindex = entry->ifindex;
if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { __clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
if (local->addr.id != msk->mpc_endpoint_id) msk->pm.local_addr_used++; }
msk->pm.extra_subflows++; found = 1; break; } rcu_read_unlock();
return found;}
static unsigned intfill_local_addresses_vec_c_flag(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals, int locals_size){ u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); u8 endp_subflow_max = mptcp_pm_get_endp_subflow_max(msk); struct sock *sk = (struct sock *)msk; struct mptcp_pm_local *local; int i = 0;
while (msk->pm.local_addr_used < endp_subflow_max) { local = &locals[i];
if (!select_local_address(pernet, msk, local)) break;
__clear_bit(local->addr.id, msk->pm.id_avail_bitmap);
if (!mptcp_pm_addr_families_match(sk, &local->addr, remote)) continue;
if (local->addr.id == msk->mpc_endpoint_id) continue;
msk->pm.local_addr_used++; msk->pm.extra_subflows++; i++;
if (msk->pm.extra_subflows >= limit_extra_subflows || i == locals_size) break; }
return i;}
static unsigned intfill_local_address_any(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *local){ struct sock *sk = (struct sock *)msk;
memset(local, 0, sizeof(*local)); local->addr.family =#if IS_ENABLED(CONFIG_MPTCP_IPV6) remote->family == AF_INET6 && ipv6_addr_v4mapped(&remote->addr6) ? AF_INET :#endif remote->family;
if (!mptcp_pm_addr_families_match(sk, &local->addr, remote)) return 0;
msk->pm.extra_subflows++;
return 1;}
/* Fill all the local addresses into the array addrs[], * and return the array size. */static unsigned intfill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *remote, struct mptcp_pm_local *locals, int locals_size){ bool c_flag_case = remote->id && mptcp_pm_add_addr_c_flag_case(msk);
/* If there is at least one MPTCP endpoint with a fullmesh flag */ if (mptcp_pm_get_endp_fullmesh_max(msk)) return fill_local_addresses_vec_fullmesh(msk, remote, locals, locals_size, c_flag_case);
/* If there is at least one MPTCP endpoint with a laminar flag */ if (mptcp_pm_get_endp_laminar_max(msk)) return fill_local_laminar_endp(msk, remote, locals);
/* Special case: peer sets the C flag, accept one ADD_ADDR if default * limits are used -- accepting no ADD_ADDR -- and use subflow endpoints */ if (c_flag_case) return fill_local_addresses_vec_c_flag(msk, remote, locals, locals_size);
/* No special case: fill in the single 'IPADDRANY' local address */ return fill_local_address_any(msk, remote, &locals[0]);}
static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk){ u8 limit_add_addr_accepted = mptcp_pm_get_limit_add_addr_accepted(msk); u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); struct mptcp_pm_local locals[MPTCP_PM_ADDR_MAX]; struct sock *sk = (struct sock *)msk; struct mptcp_addr_info remote; bool sf_created = false; int i, nr;
pr_debug("accepted %d:%d remote family %d\n", msk->pm.add_addr_accepted, limit_add_addr_accepted, msk->pm.remote.family);
remote = msk->pm.remote; mptcp_pm_announce_addr(msk, &remote, true); mptcp_pm_addr_send_ack(msk); mptcp_mpc_endpoint_setup(msk);
if (has_subflow_daddr(msk, &remote)) return;
/* pick id 0 port, if none is provided the remote address */ if (!remote.port) remote.port = sk->sk_dport;
/* connect to the specified remote address, using whatever * local address the routing configuration will pick. */ nr = fill_local_addresses_vec(msk, &remote, locals, ARRAY_SIZE(locals)); if (nr == 0) return;
spin_unlock_bh(&msk->pm.lock); for (i = 0; i < nr; i++) if (__mptcp_subflow_connect(sk, &locals[i], &remote) == 0) sf_created = true; spin_lock_bh(&msk->pm.lock);
if (sf_created) { /* add_addr_accepted is not decr for ID 0 */ if (remote.id) msk->pm.add_addr_accepted++; if (msk->pm.add_addr_accepted >= limit_add_addr_accepted || msk->pm.extra_subflows >= limit_extra_subflows) WRITE_ONCE(msk->pm.accept_addr, false); }}
void mptcp_pm_nl_rm_addr(struct mptcp_sock *msk, u8 rm_id){ if (rm_id && !WARN_ON_ONCE(msk->pm.add_addr_accepted == 0)) { u8 limit_add_addr_accepted = mptcp_pm_get_limit_add_addr_accepted(msk);
/* Note: if the subflow has been closed before, this * add_addr_accepted counter will not be decremented. */ if (--msk->pm.add_addr_accepted < limit_add_addr_accepted) WRITE_ONCE(msk->pm.accept_addr, true); }}
static bool address_use_port(struct mptcp_pm_addr_entry *entry){ return (entry->flags & (MPTCP_PM_ADDR_FLAG_SIGNAL | MPTCP_PM_ADDR_FLAG_SUBFLOW)) == MPTCP_PM_ADDR_FLAG_SIGNAL;}
/* caller must ensure the RCU grace period is already elapsed */static void __mptcp_pm_release_addr_entry(struct mptcp_pm_addr_entry *entry){ if (entry->lsk) sock_release(entry->lsk); kfree(entry);}
static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet, struct mptcp_pm_addr_entry *entry, bool replace){ struct mptcp_pm_addr_entry *cur, *del_entry = NULL; int ret = -EINVAL; u8 addr_max;
spin_lock_bh(&pernet->lock); /* to keep the code simple, don't do IDR-like allocation for address ID, * just bail when we exceed limits */ if (pernet->next_id == MPTCP_PM_MAX_ADDR_ID) pernet->next_id = 1; if (pernet->endpoints == MPTCP_PM_MAX_ADDR_ID) { ret = -ERANGE; goto out; } if (test_bit(entry->addr.id, pernet->id_bitmap)) { ret = -EBUSY; goto out; }
/* do not insert duplicate address, differentiate on port only * singled addresses */ if (!address_use_port(entry)) entry->addr.port = 0; list_for_each_entry(cur, &pernet->endp_list, list) { if (mptcp_addresses_equal(&cur->addr, &entry->addr, cur->addr.port || entry->addr.port)) { /* allow replacing the exiting endpoint only if such * endpoint is an implicit one and the user-space * did not provide an endpoint id */ if (!(cur->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)) { ret = -EEXIST; goto out; } if (entry->addr.id) goto out;
/* allow callers that only need to look up the local * addr's id to skip replacement. This allows them to * avoid calling synchronize_rcu in the packet recv * path. */ if (!replace) { kfree(entry); ret = cur->addr.id; goto out; }
pernet->endpoints--; entry->addr.id = cur->addr.id; list_del_rcu(&cur->list); del_entry = cur; break; } }
if (!entry->addr.id) {find_next: entry->addr.id = find_next_zero_bit(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1, pernet->next_id); if (!entry->addr.id && pernet->next_id != 1) { pernet->next_id = 1; goto find_next; } }
if (!entry->addr.id) goto out;
__set_bit(entry->addr.id, pernet->id_bitmap); if (entry->addr.id > pernet->next_id) pernet->next_id = entry->addr.id;
if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) { addr_max = pernet->endp_signal_max; WRITE_ONCE(pernet->endp_signal_max, addr_max + 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { addr_max = pernet->endp_subflow_max; WRITE_ONCE(pernet->endp_subflow_max, addr_max + 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) { addr_max = pernet->endp_laminar_max; WRITE_ONCE(pernet->endp_laminar_max, addr_max + 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) { addr_max = pernet->endp_fullmesh_max; WRITE_ONCE(pernet->endp_fullmesh_max, addr_max + 1); }
pernet->endpoints++; if (!entry->addr.port) list_add_tail_rcu(&entry->list, &pernet->endp_list); else list_add_rcu(&entry->list, &pernet->endp_list); ret = entry->addr.id;
out: spin_unlock_bh(&pernet->lock);
/* just replaced an existing entry, free it */ if (del_entry) { synchronize_rcu(); __mptcp_pm_release_addr_entry(del_entry); } return ret;}
static struct lock_class_key mptcp_slock_keys[2];static struct lock_class_key mptcp_keys[2];
static int mptcp_pm_nl_create_listen_socket(struct sock *sk, struct mptcp_pm_addr_entry *entry){ bool is_ipv6 = entry->addr.family == AF_INET6; int addrlen = sizeof(struct sockaddr_in); struct sockaddr_storage addr; struct sock *newsk, *ssk; int backlog = 1024; int err;
err = sock_create_kern(sock_net(sk), entry->addr.family, SOCK_STREAM, IPPROTO_MPTCP, &entry->lsk); if (err) return err;
newsk = entry->lsk->sk; if (!newsk) return -EINVAL;
/* The subflow socket lock is acquired in a nested to the msk one * in several places, even by the TCP stack, and this msk is a kernel * socket: lockdep complains. Instead of propagating the _nested * modifiers in several places, re-init the lock class for the msk * socket to an mptcp specific one. */ sock_lock_init_class_and_name(newsk, is_ipv6 ? "mlock-AF_INET6" : "mlock-AF_INET", &mptcp_slock_keys[is_ipv6], is_ipv6 ? "msk_lock-AF_INET6" : "msk_lock-AF_INET", &mptcp_keys[is_ipv6]);
lock_sock(newsk); ssk = __mptcp_nmpc_sk(mptcp_sk(newsk)); release_sock(newsk); if (IS_ERR(ssk)) return PTR_ERR(ssk);
mptcp_info2sockaddr(&entry->addr, &addr, entry->addr.family);#if IS_ENABLED(CONFIG_MPTCP_IPV6) if (entry->addr.family == AF_INET6) addrlen = sizeof(struct sockaddr_in6);#endif if (ssk->sk_family == AF_INET) err = inet_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);#if IS_ENABLED(CONFIG_MPTCP_IPV6) else if (ssk->sk_family == AF_INET6) err = inet6_bind_sk(ssk, (struct sockaddr_unsized *)&addr, addrlen);#endif if (err) return err;
/* We don't use mptcp_set_state() here because it needs to be called * under the msk socket lock. For the moment, that will not bring * anything more than only calling inet_sk_state_store(), because the * old status is known (TCP_CLOSE). */ inet_sk_state_store(newsk, TCP_LISTEN); lock_sock(ssk); WRITE_ONCE(mptcp_subflow_ctx(ssk)->pm_listener, true); err = __inet_listen_sk(ssk, backlog); if (!err) mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED); release_sock(ssk); return err;}
int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_pm_addr_entry *skc){ struct mptcp_pm_addr_entry *entry; struct pm_nl_pernet *pernet; int ret;
pernet = pm_nl_get_pernet_from_msk(msk);
rcu_read_lock(); entry = __lookup_addr(pernet, &skc->addr); ret = entry ? entry->addr.id : -1; rcu_read_unlock(); if (ret >= 0) return ret;
/* address not found, add to local list */ entry = kmemdup(skc, sizeof(*skc), GFP_ATOMIC); if (!entry) return -ENOMEM;
entry->addr.port = 0; ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, false); if (ret < 0) kfree(entry);
return ret;}
bool mptcp_pm_nl_is_backup(struct mptcp_sock *msk, struct mptcp_addr_info *skc){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk); struct mptcp_pm_addr_entry *entry; bool backup;
rcu_read_lock(); entry = __lookup_addr(pernet, skc); backup = entry && !!(entry->flags & MPTCP_PM_ADDR_FLAG_BACKUP); rcu_read_unlock();
return backup;}
static int mptcp_nl_add_subflow_or_signal_addr(struct net *net, struct mptcp_addr_info *addr){ struct mptcp_sock *msk; long s_slot = 0, s_num = 0;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; struct mptcp_addr_info mpc_addr;
if (!READ_ONCE(msk->fully_established) || mptcp_pm_is_userspace(msk)) goto next;
/* if the endp linked to the init sf is re-added with a != ID */ mptcp_local_address((struct sock_common *)msk, &mpc_addr);
lock_sock(sk); spin_lock_bh(&msk->pm.lock); if (mptcp_addresses_equal(addr, &mpc_addr, addr->port)) msk->mpc_endpoint_id = addr->id; mptcp_pm_create_subflow_or_signal_addr(msk); spin_unlock_bh(&msk->pm.lock); release_sock(sk);
next: sock_put(sk); cond_resched(); }
return 0;}
/* Add an MPTCP endpoint */int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct mptcp_pm_addr_entry addr, *entry; struct nlattr *attr; int ret;
if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR)) return -EINVAL;
attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR]; ret = mptcp_pm_parse_entry(attr, info, true, &addr); if (ret < 0) return ret;
if (addr.addr.port && !address_use_port(&addr)) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "flags must have signal and not subflow when using port"); return -EINVAL; }
if (addr.flags & MPTCP_PM_ADDR_FLAG_SIGNAL && addr.flags & MPTCP_PM_ADDR_FLAG_FULLMESH) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "flags mustn't have both signal and fullmesh"); return -EINVAL; }
if (addr.flags & MPTCP_PM_ADDR_FLAG_IMPLICIT) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "can't create IMPLICIT endpoint"); return -EINVAL; }
entry = kmemdup(&addr, sizeof(addr), GFP_KERNEL_ACCOUNT); if (!entry) { GENL_SET_ERR_MSG(info, "can't allocate addr"); return -ENOMEM; }
if (entry->addr.port) { ret = mptcp_pm_nl_create_listen_socket(skb->sk, entry); if (ret) { GENL_SET_ERR_MSG_FMT(info, "create listen socket error: %d", ret); goto out_free; } } ret = mptcp_pm_nl_append_new_local_addr(pernet, entry, true); if (ret < 0) { GENL_SET_ERR_MSG_FMT(info, "too many addresses or duplicate one: %d", ret); goto out_free; }
mptcp_nl_add_subflow_or_signal_addr(sock_net(skb->sk), &entry->addr); return 0;
out_free: __mptcp_pm_release_addr_entry(entry); return ret;}
static void mptcp_pm_remove_announced(struct mptcp_sock *msk, const struct mptcp_addr_info *addr, bool force){ struct mptcp_rm_list list = { .nr = 0 }; bool announced;
list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
announced = mptcp_pm_announced_remove(msk, addr); if (announced || force) { spin_lock_bh(&msk->pm.lock); if (announced) msk->pm.add_addr_signaled--; mptcp_pm_remove_addr(msk, &list); spin_unlock_bh(&msk->pm.lock); }}
static void __mark_subflow_endp_available(struct mptcp_sock *msk, u8 id){ /* If it was marked as used, and not ID 0, decrement local_addr_used */ if (!__test_and_set_bit(id ? : msk->mpc_endpoint_id, msk->pm.id_avail_bitmap) && id && !WARN_ON_ONCE(msk->pm.local_addr_used == 0)) msk->pm.local_addr_used--;}
static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net, const struct mptcp_pm_addr_entry *entry){ const struct mptcp_addr_info *addr = &entry->addr; struct mptcp_rm_list list = { .nr = 1 }; long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
pr_debug("remove_id=%d\n", addr->id);
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; bool remove_subflow;
if (mptcp_pm_is_userspace(msk)) goto next;
lock_sock(sk); remove_subflow = mptcp_pm_has_subflow_saddr(msk, addr); mptcp_pm_remove_announced(msk, addr, remove_subflow && !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT));
list.ids[0] = mptcp_endp_get_local_id(msk, addr);
spin_lock_bh(&msk->pm.lock); if (remove_subflow) mptcp_pm_rm_subflow(msk, &list); if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) __mark_subflow_endp_available(msk, list.ids[0]); else /* mark endp ID as available, e.g. Signal or MPC endp */ __set_bit(addr->id, msk->pm.id_avail_bitmap); spin_unlock_bh(&msk->pm.lock);
if (msk->mpc_endpoint_id == entry->addr.id) msk->mpc_endpoint_id = 0; release_sock(sk);
next: sock_put(sk); cond_resched(); }
return 0;}
static int mptcp_nl_remove_id_zero_address(struct net *net, struct mptcp_addr_info *addr){ struct mptcp_rm_list list = { .nr = 0 }; long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
list.ids[list.nr++] = 0;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; struct mptcp_addr_info msk_local; struct mptcp_addr_info anno_addr; bool announced;
if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk)) goto next;
mptcp_local_address((struct sock_common *)msk, &msk_local); if (!mptcp_addresses_equal(&msk_local, addr, addr->port)) goto next;
lock_sock(sk); /* Drop a possibly pending ADD_ADDR for this address. */ anno_addr = msk_local; anno_addr.port = 0; announced = mptcp_pm_announced_remove(msk, &anno_addr); spin_lock_bh(&msk->pm.lock); if (announced) msk->pm.add_addr_signaled--; mptcp_pm_remove_addr(msk, &list); mptcp_pm_rm_subflow(msk, &list); __mark_subflow_endp_available(msk, 0); spin_unlock_bh(&msk->pm.lock); release_sock(sk);
next: sock_put(sk); cond_resched(); }
return 0;}
/* Remove an MPTCP endpoint */int mptcp_pm_nl_del_addr_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct mptcp_pm_addr_entry addr, *entry; struct nlattr *attr; u8 addr_max; int ret;
if (GENL_REQ_ATTR_CHECK(info, MPTCP_PM_ENDPOINT_ADDR)) return -EINVAL;
attr = info->attrs[MPTCP_PM_ENDPOINT_ADDR]; ret = mptcp_pm_parse_entry(attr, info, false, &addr); if (ret < 0) return ret;
/* the zero id address is special: the first address used by the msk * always gets such an id, so different subflows can have different zero * id addresses. Additionally zero id is not accounted for in id_bitmap. * Let's use an 'mptcp_rm_list' instead of the common remove code. */ if (addr.addr.id == 0) return mptcp_nl_remove_id_zero_address(sock_net(skb->sk), &addr.addr);
spin_lock_bh(&pernet->lock); entry = __lookup_addr_by_id(pernet, addr.addr.id); if (!entry) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found"); spin_unlock_bh(&pernet->lock); return -EINVAL; } if (entry->flags & MPTCP_PM_ADDR_FLAG_SIGNAL) { addr_max = pernet->endp_signal_max; WRITE_ONCE(pernet->endp_signal_max, addr_max - 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { addr_max = pernet->endp_subflow_max; WRITE_ONCE(pernet->endp_subflow_max, addr_max - 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_LAMINAR) { addr_max = pernet->endp_laminar_max; WRITE_ONCE(pernet->endp_laminar_max, addr_max - 1); } if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) { addr_max = pernet->endp_fullmesh_max; WRITE_ONCE(pernet->endp_fullmesh_max, addr_max - 1); }
pernet->endpoints--; list_del_rcu(&entry->list); __clear_bit(entry->addr.id, pernet->id_bitmap); spin_unlock_bh(&pernet->lock);
mptcp_nl_remove_subflow_and_signal_addr(sock_net(skb->sk), entry); synchronize_rcu(); __mptcp_pm_release_addr_entry(entry);
return ret;}
static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk, struct list_head *rm_list, struct mptcp_pm_addr_entry *entry){ struct mptcp_rm_list alist, slist; bool more;
again: alist.nr = 0; slist.nr = 0; more = false;
entry = list_prepare_entry(entry, rm_list, list); list_for_each_entry_continue(entry, rm_list, list) { if (mptcp_pm_has_subflow_saddr(msk, &entry->addr)) slist.ids[slist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
if (mptcp_pm_announced_remove(msk, &entry->addr)) alist.ids[alist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
if (slist.nr == MPTCP_RM_IDS_MAX || alist.nr == MPTCP_RM_IDS_MAX) { more = !list_is_last(&entry->list, rm_list); break; } }
spin_lock_bh(&msk->pm.lock); if (alist.nr) { msk->pm.add_addr_signaled -= alist.nr; mptcp_pm_remove_addr(msk, &alist); } if (slist.nr) mptcp_pm_rm_subflow(msk, &slist); /* Reset counters: maybe some subflows have been removed before */ if (!more) { bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); msk->pm.local_addr_used = 0; } spin_unlock_bh(&msk->pm.lock);
if (more) goto again;}
static void mptcp_nl_flush_addrs_list(struct net *net, struct list_head *rm_list){ long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
if (list_empty(rm_list)) return;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk;
if (!mptcp_pm_is_userspace(msk)) { lock_sock(sk); mptcp_pm_flush_addrs_and_subflows(msk, rm_list, NULL); release_sock(sk); }
sock_put(sk); cond_resched(); }}
/* caller must ensure the RCU grace period is already elapsed */static void __flush_addrs(struct list_head *list){ while (!list_empty(list)) { struct mptcp_pm_addr_entry *cur;
cur = list_entry(list->next, struct mptcp_pm_addr_entry, list); list_del_rcu(&cur->list); __mptcp_pm_release_addr_entry(cur); }}
static void __reset_counters(struct pm_nl_pernet *pernet){ WRITE_ONCE(pernet->endp_signal_max, 0); WRITE_ONCE(pernet->endp_subflow_max, 0); WRITE_ONCE(pernet->endp_laminar_max, 0); WRITE_ONCE(pernet->endp_fullmesh_max, 0); pernet->endpoints = 0;}
int mptcp_pm_nl_flush_addrs_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct list_head free_list;
spin_lock_bh(&pernet->lock); free_list = pernet->endp_list; INIT_LIST_HEAD_RCU(&pernet->endp_list); __reset_counters(pernet); pernet->next_id = 1; bitmap_zero(pernet->id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); spin_unlock_bh(&pernet->lock);
if (free_list.next == &pernet->endp_list) return 0;
synchronize_rcu();
/* Adjust the pointers to free_list instead of pernet->endp_list */ free_list.prev->next = &free_list; free_list.next->prev = &free_list;
mptcp_nl_flush_addrs_list(sock_net(skb->sk), &free_list); __flush_addrs(&free_list); return 0;}
int mptcp_pm_nl_get_addr(u8 id, struct mptcp_pm_addr_entry *addr, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct mptcp_pm_addr_entry *entry; int ret = -EINVAL;
rcu_read_lock(); entry = __lookup_addr_by_id(pernet, id); if (entry) { *addr = *entry; ret = 0; } rcu_read_unlock();
return ret;}
int mptcp_pm_nl_dump_addr(struct sk_buff *msg, struct netlink_callback *cb){ struct net *net = sock_net(msg->sk); struct mptcp_pm_addr_entry *entry; struct pm_nl_pernet *pernet; int id = cb->args[0]; int i;
pernet = pm_nl_get_pernet(net);
rcu_read_lock(); for (i = id; i < MPTCP_PM_MAX_ADDR_ID + 1; i++) { if (test_bit(i, pernet->id_bitmap)) { entry = __lookup_addr_by_id(pernet, i); if (!entry) break;
if (entry->addr.id <= id) continue;
if (mptcp_pm_genl_fill_addr(msg, cb, entry) < 0) break;
id = entry->addr.id; } } rcu_read_unlock();
cb->args[0] = id; return msg->len;}
static int parse_limit(struct genl_info *info, int id, unsigned int *limit){ struct nlattr *attr = info->attrs[id];
if (!attr) return 0;
*limit = nla_get_u32(attr); if (*limit > MPTCP_PM_SUBFLOWS_MAX) { NL_SET_ERR_MSG_ATTR_FMT(info->extack, attr, "limit greater than maximum (%u)", MPTCP_PM_SUBFLOWS_MAX); return -EINVAL; } return 0;}
int mptcp_pm_nl_set_limits_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); unsigned int rcv_addrs, subflows; int ret;
spin_lock_bh(&pernet->lock); rcv_addrs = pernet->limit_add_addr_accepted; ret = parse_limit(info, MPTCP_PM_ATTR_RCV_ADD_ADDRS, &rcv_addrs); if (ret) goto unlock;
subflows = pernet->limit_extra_subflows; ret = parse_limit(info, MPTCP_PM_ATTR_SUBFLOWS, &subflows); if (ret) goto unlock;
WRITE_ONCE(pernet->limit_add_addr_accepted, rcv_addrs); WRITE_ONCE(pernet->limit_extra_subflows, subflows);
unlock: spin_unlock_bh(&pernet->lock); return ret;}
int mptcp_pm_nl_get_limits_doit(struct sk_buff *skb, struct genl_info *info){ struct pm_nl_pernet *pernet = genl_info_pm_nl(info); struct sk_buff *msg; void *reply;
msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); if (!msg) return -ENOMEM;
reply = genlmsg_put_reply(msg, info, &mptcp_genl_family, 0, MPTCP_PM_CMD_GET_LIMITS); if (!reply) goto fail;
if (nla_put_u32(msg, MPTCP_PM_ATTR_RCV_ADD_ADDRS, READ_ONCE(pernet->limit_add_addr_accepted))) goto fail;
if (nla_put_u32(msg, MPTCP_PM_ATTR_SUBFLOWS, READ_ONCE(pernet->limit_extra_subflows))) goto fail;
genlmsg_end(msg, reply); return genlmsg_reply(msg, info);
fail: GENL_SET_ERR_MSG(info, "not enough space in Netlink message"); nlmsg_free(msg); return -EMSGSIZE;}
static void mptcp_pm_nl_fullmesh(struct mptcp_sock *msk, struct mptcp_addr_info *addr){ struct mptcp_rm_list list = { .nr = 0 };
list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr);
spin_lock_bh(&msk->pm.lock); mptcp_pm_rm_subflow(msk, &list); __mark_subflow_endp_available(msk, list.ids[0]); mptcp_pm_create_subflow_or_signal_addr(msk); spin_unlock_bh(&msk->pm.lock);}
static void mptcp_pm_nl_set_flags_all(struct net *net, struct mptcp_pm_addr_entry *local, u8 changed){ u8 is_subflow = !!(local->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW); u8 bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP); long s_slot = 0, s_num = 0; struct mptcp_sock *msk;
if (changed == MPTCP_PM_ADDR_FLAG_FULLMESH && !is_subflow) return;
while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk;
if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk)) goto next;
lock_sock(sk); if (changed & MPTCP_PM_ADDR_FLAG_BACKUP) mptcp_pm_mp_prio_send_ack(msk, &local->addr, NULL, bkup); /* Subflows will only be recreated if the SUBFLOW flag is set */ if (is_subflow && (changed & MPTCP_PM_ADDR_FLAG_FULLMESH)) mptcp_pm_nl_fullmesh(msk, &local->addr); release_sock(sk);
next: sock_put(sk); cond_resched(); }}
int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local, struct genl_info *info){ struct nlattr *attr = info->attrs[MPTCP_PM_ATTR_ADDR]; u8 changed, mask = MPTCP_PM_ADDR_FLAG_BACKUP | MPTCP_PM_ADDR_FLAG_FULLMESH; struct net *net = genl_info_net(info); struct mptcp_pm_addr_entry *entry; struct pm_nl_pernet *pernet; u8 lookup_by_id = 0;
pernet = pm_nl_get_pernet(net);
if (local->addr.family == AF_UNSPEC) { lookup_by_id = 1; if (!local->addr.id) { NL_SET_ERR_MSG_ATTR(info->extack, attr, "missing address ID"); return -EOPNOTSUPP; } }
spin_lock_bh(&pernet->lock); entry = lookup_by_id ? __lookup_addr_by_id(pernet, local->addr.id) : __lookup_addr(pernet, &local->addr); if (!entry) { spin_unlock_bh(&pernet->lock); NL_SET_ERR_MSG_ATTR(info->extack, attr, "address not found"); return -EINVAL; } if ((local->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) && (entry->flags & (MPTCP_PM_ADDR_FLAG_SIGNAL | MPTCP_PM_ADDR_FLAG_IMPLICIT))) { spin_unlock_bh(&pernet->lock); NL_SET_ERR_MSG_ATTR(info->extack, attr, "invalid addr flags"); return -EINVAL; }
changed = (local->flags ^ entry->flags) & mask; entry->flags = (entry->flags & ~mask) | (local->flags & mask); *local = *entry;
if (changed & MPTCP_PM_ADDR_FLAG_FULLMESH) { u8 addr_max = pernet->endp_fullmesh_max;
if (entry->flags & MPTCP_PM_ADDR_FLAG_FULLMESH) addr_max++; else addr_max--;
WRITE_ONCE(pernet->endp_fullmesh_max, addr_max); }
spin_unlock_bh(&pernet->lock);
mptcp_pm_nl_set_flags_all(net, local, changed); return 0;}
bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk){ struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
if (msk->pm.extra_subflows == mptcp_pm_get_limit_extra_subflows(msk) || (find_next_and_bit(pernet->id_bitmap, msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1, 0) == MPTCP_PM_MAX_ADDR_ID + 1)) { WRITE_ONCE(msk->pm.work_pending, false); return false; } return true;}
/* Called under PM lock */void __mptcp_pm_kernel_worker(struct mptcp_sock *msk){ struct mptcp_pm_data *pm = &msk->pm;
if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) { pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED); mptcp_pm_nl_add_addr_received(msk); } if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) { pm->status &= ~BIT(MPTCP_PM_ESTABLISHED); mptcp_pm_nl_fully_established(msk); } if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) { pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED); mptcp_pm_nl_subflow_established(msk); }}
static int __net_init pm_nl_init_net(struct net *net){ struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
INIT_LIST_HEAD_RCU(&pernet->endp_list);
/* Cit. 2 subflows ought to be enough for anybody. */ pernet->limit_extra_subflows = 2; pernet->next_id = 1; spin_lock_init(&pernet->lock);
/* No need to initialize other pernet fields, the struct is zeroed at * allocation time. */
return 0;}
static void __net_exit pm_nl_exit_net(struct list_head *net_list){ struct net *net;
list_for_each_entry(net, net_list, exit_list) { struct pm_nl_pernet *pernet = pm_nl_get_pernet(net);
/* net is removed from namespace list, can't race with * other modifiers, also netns core already waited for a * RCU grace period. */ __flush_addrs(&pernet->endp_list); }}
static struct pernet_operations mptcp_pm_pernet_ops = { .init = pm_nl_init_net, .exit_batch = pm_nl_exit_net, .id = &pm_nl_pernet_id, .size = sizeof(struct pm_nl_pernet),};
struct mptcp_pm_ops mptcp_pm_kernel = { .name = "kernel", .owner = THIS_MODULE,};
void __init mptcp_pm_kernel_register(void){ if (register_pernet_subsys(&mptcp_pm_pernet_ops) < 0) panic("Failed to register MPTCP PM pernet subsystem.\n");
mptcp_pm_register(&mptcp_pm_kernel);}