Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef _NET_GRO_H#define _NET_GRO_H
#include <linux/indirect_call_wrapper.h>#include <linux/ip.h>#include <linux/ipv6.h>#include <net/ip6_checksum.h>#include <linux/skbuff.h>#include <net/udp.h>#include <net/hotdata.h>
/* This should be increased if a protocol with a bigger head is added. */#define GRO_MAX_HEAD (MAX_HEADER + 128)
struct napi_gro_cb { union { struct { /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ void *frag0;
/* Length of frag0. */ unsigned int frag0_len; };
struct { /* used in skb_gro_receive() slow path */ struct sk_buff *last;
/* jiffies when first packet was created/queued */ unsigned long age; }; };
/* This indicates where we are processing relative to skb->data. */ int data_offset;
/* This is non-zero if the packet cannot be merged with the new skb. */ u16 flush;
/* Number of segments aggregated. */ u16 count;
/* Used in ipv6_gro_receive() and foo-over-udp and esp-in-udp */ u16 proto;
u16 pad;
/* Used in napi_gro_cb::free */#define NAPI_GRO_FREE 1#define NAPI_GRO_FREE_STOLEN_HEAD 2 /* portion of the cb set to zero at every gro iteration */ struct_group(zeroed,
/* Start offset for remote checksum offload */ u16 gro_remcsum_start;
/* This is non-zero if the packet may be of the same flow. */ u8 same_flow:1;
/* Used in tunnel GRO receive */ u8 encap_mark:1;
/* GRO checksum is valid */ u8 csum_valid:1;
/* Number of checksums via CHECKSUM_UNNECESSARY */ u8 csum_cnt:3;
/* Free the skb? */ u8 free:2;
/* Used in GRE, set in fou/gue_gro_receive */ u8 is_fou:1;
/* Used to determine if ipid_offset can be ignored */ u8 ip_fixedid:2;
/* Number of gro_receive callbacks this packet already went through */ u8 recursion_counter:4;
/* GRO is done by frag_list pointer chaining. */ u8 is_flist:1; );
/* used to support CHECKSUM_COMPLETE for tunneling protocols */ __wsum csum;
/* L3 offsets */ union { struct { u16 network_offset; u16 inner_network_offset; }; u16 network_offsets[2]; };};
#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
#define GRO_RECURSION_LIMIT 15static inline int gro_recursion_inc_test(struct sk_buff *skb){ return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;}
typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);static inline struct sk_buff *call_gro_receive(gro_receive_t cb, struct list_head *head, struct sk_buff *skb){ if (unlikely(gro_recursion_inc_test(skb))) { NAPI_GRO_CB(skb)->flush |= 1; return NULL; }
return cb(head, skb);}
typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, struct sk_buff *);static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, struct sock *sk, struct list_head *head, struct sk_buff *skb){ if (unlikely(gro_recursion_inc_test(skb))) { NAPI_GRO_CB(skb)->flush |= 1; return NULL; }
return cb(sk, head, skb);}
static inline unsigned int skb_gro_offset(const struct sk_buff *skb){ return NAPI_GRO_CB(skb)->data_offset;}
static inline unsigned int skb_gro_len(const struct sk_buff *skb){ return skb->len - NAPI_GRO_CB(skb)->data_offset;}
static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len){ NAPI_GRO_CB(skb)->data_offset += len;}
static inline void *skb_gro_header_fast(const struct sk_buff *skb, unsigned int offset){ return NAPI_GRO_CB(skb)->frag0 + offset;}
static inline bool skb_gro_may_pull(const struct sk_buff *skb, unsigned int hlen){ return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);}
static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, unsigned int offset){ if (!pskb_may_pull(skb, hlen)) return NULL;
return skb->data + offset;}
static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen, unsigned int offset){ void *ptr;
ptr = skb_gro_header_fast(skb, offset); if (!skb_gro_may_pull(skb, hlen)) ptr = skb_gro_header_slow(skb, hlen, offset); return ptr;}
static inline int skb_gro_receive_network_offset(const struct sk_buff *skb){ return NAPI_GRO_CB(skb)->network_offsets[NAPI_GRO_CB(skb)->encap_mark];}
static inline void *skb_gro_network_header(const struct sk_buff *skb){ if (skb_gro_may_pull(skb, skb_gro_offset(skb))) return skb_gro_header_fast(skb, skb_gro_receive_network_offset(skb));
return skb->data + skb_gro_receive_network_offset(skb);}
static inline __wsum inet_gro_compute_pseudo(const struct sk_buff *skb, int proto){ const struct iphdr *iph = skb_gro_network_header(skb);
return csum_tcpudp_nofold(iph->saddr, iph->daddr, skb_gro_len(skb), proto, 0);}
static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len){ if (NAPI_GRO_CB(skb)->csum_valid) NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len, wsum_negate(NAPI_GRO_CB(skb)->csum)));}
/* GRO checksum functions. These are logical equivalents of the normal * checksum functions (in skbuff.h) except that they operate on the GRO * offsets and fields in sk_buff. */
__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb){ return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));}
static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, bool zero_okay, __sum16 check){ return ((skb->ip_summed != CHECKSUM_PARTIAL || skb_checksum_start_offset(skb) < skb_gro_offset(skb)) && !skb_at_gro_remcsum_start(skb) && NAPI_GRO_CB(skb)->csum_cnt == 0 && (!zero_okay || check));}
static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, __wsum psum){ if (NAPI_GRO_CB(skb)->csum_valid && !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) return 0;
NAPI_GRO_CB(skb)->csum = psum;
return __skb_gro_checksum_complete(skb);}
static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb){ if (NAPI_GRO_CB(skb)->csum_cnt > 0) { /* Consume a checksum from CHECKSUM_UNNECESSARY */ NAPI_GRO_CB(skb)->csum_cnt--; } else { /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we * verified a new top level checksum or an encapsulated one * during GRO. This saves work if we fallback to normal path. */ __skb_incr_checksum_unnecessary(skb); }}
#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ compute_pseudo) \({ \ __sum16 __ret = 0; \ if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ __ret = __skb_gro_checksum_validate_complete(skb, \ compute_pseudo(skb, proto)); \ if (!__ret) \ skb_gro_incr_csum_unnecessary(skb); \ __ret; \})
#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
#define skb_gro_checksum_validate_zero_check(skb, proto, check, \ compute_pseudo) \ __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
#define skb_gro_checksum_simple_validate(skb) \ __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb){ return (NAPI_GRO_CB(skb)->csum_cnt == 0 && !NAPI_GRO_CB(skb)->csum_valid);}
static inline void __skb_gro_checksum_convert(struct sk_buff *skb, __wsum pseudo){ NAPI_GRO_CB(skb)->csum = ~pseudo; NAPI_GRO_CB(skb)->csum_valid = 1;}
#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \do { \ if (__skb_gro_checksum_convert_check(skb)) \ __skb_gro_checksum_convert(skb, \ compute_pseudo(skb, proto)); \} while (0)
struct gro_remcsum { int offset; __wsum delta;};
static inline void skb_gro_remcsum_init(struct gro_remcsum *grc){ grc->offset = 0; grc->delta = 0;}
static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, unsigned int off, size_t hdrlen, int start, int offset, struct gro_remcsum *grc, bool nopartial){ __wsum delta; size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
if (!nopartial) { NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; return ptr; }
ptr = skb_gro_header(skb, off + plen, off); if (!ptr) return NULL;
delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, start, offset);
/* Adjust skb->csum since we changed the packet */ NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
grc->offset = off + hdrlen + offset; grc->delta = delta;
return ptr;}
static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, struct gro_remcsum *grc){ void *ptr; size_t plen = grc->offset + sizeof(u16);
if (!grc->delta) return;
ptr = skb_gro_header(skb, plen, grc->offset); if (!ptr) return;
remcsum_unadjust((__sum16 *)ptr, grc->delta);}
#ifdef CONFIG_XFRM_OFFLOADstatic inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush){ if (PTR_ERR(pp) != -EINPROGRESS) NAPI_GRO_CB(skb)->flush |= flush;}static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, struct sk_buff *pp, int flush, struct gro_remcsum *grc){ if (PTR_ERR(pp) != -EINPROGRESS) { NAPI_GRO_CB(skb)->flush |= flush; skb_gro_remcsum_cleanup(skb, grc); skb->remcsum_offload = 0; }}#elsestatic inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush){ NAPI_GRO_CB(skb)->flush |= flush;}static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, struct sk_buff *pp, int flush, struct gro_remcsum *grc){ NAPI_GRO_CB(skb)->flush |= flush; skb_gro_remcsum_cleanup(skb, grc); skb->remcsum_offload = 0;}#endif
INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, struct sk_buff *));INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, struct sk_buff *));INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, struct sk_buff *));INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
struct sk_buff *udp6_gro_receive(struct list_head *, struct sk_buff *);int udp6_gro_complete(struct sk_buff *, int);
#define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \({ \ unlikely(gro_recursion_inc_test(skb)) ? \ NAPI_GRO_CB(skb)->flush |= 1, NULL : \ INDIRECT_CALL_INET(cb, f2, f1, head, skb); \})
struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, struct udphdr *uh, struct sock *sk);int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb){ struct udphdr *uh; unsigned int hlen, off;
off = skb_gro_offset(skb); hlen = off + sizeof(*uh); uh = skb_gro_header(skb, hlen, off);
return uh;}
static inline __wsum ip6_gro_compute_pseudo(const struct sk_buff *skb, int proto){ const struct ipv6hdr *iph = skb_gro_network_header(skb);
return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, skb_gro_len(skb), proto, 0));}
static inline int inet_gro_flush(const struct iphdr *iph, const struct iphdr *iph2, struct sk_buff *p, bool inner){ const u32 id = ntohl(*(__be32 *)&iph->id); const u32 id2 = ntohl(*(__be32 *)&iph2->id); const u16 ipid_offset = (id >> 16) - (id2 >> 16); const u16 count = NAPI_GRO_CB(p)->count;
/* All fields must match except length and checksum. */ if ((iph->ttl ^ iph2->ttl) | (iph->tos ^ iph2->tos) | ((id ^ id2) & IP_DF)) return true;
/* When we receive our second frame we can make a decision on if we * continue this flow as an atomic flow with a fixed ID or if we use * an incrementing ID. */ if (count == 1 && !ipid_offset) NAPI_GRO_CB(p)->ip_fixedid |= 1 << inner;
return ipid_offset ^ (count * !(NAPI_GRO_CB(p)->ip_fixedid & (1 << inner)));}
static inline int ipv6_gro_flush(const struct ipv6hdr *iph, const struct ipv6hdr *iph2){ /* <Version:4><Traffic_Class:8><Flow_Label:20> */ __be32 first_word = *(__be32 *)iph ^ *(__be32 *)iph2;
/* Flush if Traffic Class fields are different. */ return !!((first_word & htonl(0x0FF00000)) | (__force __be32)(iph->hop_limit ^ iph2->hop_limit));}
static inline int __gro_receive_network_flush(const void *th, const void *th2, struct sk_buff *p, const u16 diff, bool inner){ const void *nh = th - diff; const void *nh2 = th2 - diff;
if (((struct iphdr *)nh)->version == 6) return ipv6_gro_flush(nh, nh2); else return inet_gro_flush(nh, nh2, p, inner);}
static inline int gro_receive_network_flush(const void *th, const void *th2, struct sk_buff *p){ int off = skb_transport_offset(p); int flush;
flush = __gro_receive_network_flush(th, th2, p, off - NAPI_GRO_CB(p)->network_offset, false); if (NAPI_GRO_CB(p)->encap_mark) flush |= __gro_receive_network_flush(th, th2, p, off - NAPI_GRO_CB(p)->inner_network_offset, true);
return flush;}
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb);void __gro_flush(struct gro_node *gro, bool flush_old);
static inline void gro_flush(struct gro_node *gro, bool flush_old){ if (!gro->bitmask) return;
__gro_flush(gro, flush_old);}
static inline void napi_gro_flush(struct napi_struct *napi, bool flush_old){ gro_flush(&napi->gro, flush_old);}
/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */static inline void gro_normal_list(struct gro_node *gro){ if (!gro->rx_count) return; netif_receive_skb_list_internal(&gro->rx_list); INIT_LIST_HEAD(&gro->rx_list); gro->rx_count = 0;}
static inline void gro_flush_normal(struct gro_node *gro, bool flush_old){ gro_flush(gro, flush_old); gro_normal_list(gro);}
/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, * pass the whole batch up to the stack. */static inline void gro_normal_one(struct gro_node *gro, struct sk_buff *skb, int segs){ list_add_tail(&skb->list, &gro->rx_list); gro->rx_count += segs; if (gro->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch)) gro_normal_list(gro);}
void gro_init(struct gro_node *gro);void gro_cleanup(struct gro_node *gro);
/* This function is the alternative of 'inet_iif' and 'inet_sdif' * functions in case we can not rely on fields of IPCB. * * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized. * The caller must hold the RCU read lock. */static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif){ *iif = inet_iif(skb) ?: skb->dev->ifindex; *sdif = 0;
#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) if (netif_is_l3_slave(skb->dev)) { struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);
*sdif = *iif; *iif = master ? master->ifindex : 0; }#endif}
/* This function is the alternative of 'inet6_iif' and 'inet6_sdif' * functions in case we can not rely on fields of IP6CB. * * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized. * The caller must hold the RCU read lock. */static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif){ /* using skb->dev->ifindex because skb_dst(skb) is not initialized */ *iif = skb->dev->ifindex; *sdif = 0;
#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) if (netif_is_l3_slave(skb->dev)) { struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev);
*sdif = *iif; *iif = master ? master->ifindex : 0; }#endif}
struct packet_offload *gro_find_receive_by_type(__be16 type);struct packet_offload *gro_find_complete_by_type(__be16 type);
static inline struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb){ unsigned int thlen, hlen, off; struct tcphdr *th;
off = skb_gro_offset(skb); hlen = off + sizeof(*th); th = skb_gro_header(skb, hlen, off); if (unlikely(!th)) return NULL;
thlen = th->doff * 4; if (unlikely(thlen < sizeof(*th))) return NULL;
hlen = off + thlen; if (!skb_gro_may_pull(skb, hlen)) { th = skb_gro_header_slow(skb, hlen, off); if (unlikely(!th)) return NULL; }
skb_gro_pull(skb, thlen);
return th;}
#endif /* _NET_GRO_H */