Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553/* SPDX-License-Identifier: GPL-2.0 */#ifndef __NET_SCHED_GENERIC_H#define __NET_SCHED_GENERIC_H
#include <linux/netdevice.h>#include <linux/types.h>#include <linux/rcupdate.h>#include <linux/pkt_sched.h>#include <linux/pkt_cls.h>#include <linux/percpu.h>#include <linux/dynamic_queue_limits.h>#include <linux/list.h>#include <linux/refcount.h>#include <linux/workqueue.h>#include <linux/mutex.h>#include <linux/rwsem.h>#include <linux/atomic.h>#include <linux/hashtable.h>#include <net/gen_stats.h>#include <net/rtnetlink.h>#include <net/flow_offload.h>#include <linux/xarray.h>#include <net/dropreason-qdisc.h>
struct Qdisc_ops;struct qdisc_walker;struct tcf_walker;struct module;struct bpf_flow_keys;struct Qdisc;struct netdev_queue;
struct qdisc_rate_table { struct tc_ratespec rate; u32 data[256]; struct qdisc_rate_table *next; int refcnt;};
enum qdisc_state_t { __QDISC_STATE_SCHED, __QDISC_STATE_DEACTIVATED, __QDISC_STATE_MISSED, __QDISC_STATE_DRAINING,};
#define QDISC_STATE_MISSED BIT(__QDISC_STATE_MISSED)#define QDISC_STATE_DRAINING BIT(__QDISC_STATE_DRAINING)
#define QDISC_STATE_NON_EMPTY (QDISC_STATE_MISSED | \ QDISC_STATE_DRAINING)
struct qdisc_size_table { struct rcu_head rcu; struct list_head list; struct tc_sizespec szopts; int refcnt; u16 data[];};
/* similar to sk_buff_head, but skb->prev pointer is undefined. */struct qdisc_skb_head { struct sk_buff *head; struct sk_buff *tail; __u32 qlen; spinlock_t lock;};
struct Qdisc { int (*enqueue)(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free); struct sk_buff * (*dequeue)(struct Qdisc *sch); unsigned int flags;#define TCQ_F_BUILTIN 1#define TCQ_F_INGRESS 2#define TCQ_F_CAN_BYPASS 4#define TCQ_F_MQROOT 8#define TCQ_F_ONETXQUEUE 0x10 /* dequeue_skb() can assume all skbs are for * q->dev_queue : It can test * netif_xmit_frozen_or_stopped() before * dequeueing next packet. * Its true for MQ/MQPRIO slaves, or non * multiqueue device. */#define TCQ_F_WARN_NONWC (1 << 16)#define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */#define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : * qdisc_tree_reduce_backlog() should stop. */#define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */#define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */#define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */#define TCQ_F_DEQUEUE_DROPS 0x400 /* ->dequeue() can drop packets in q->to_free */
u32 limit; const struct Qdisc_ops *ops; struct qdisc_size_table __rcu *stab; struct hlist_node hash; u32 handle; u32 parent; int depth;
struct netdev_queue *dev_queue;
struct net_rate_estimator __rcu *rate_est; struct gnet_stats_basic_sync __percpu *cpu_bstats; struct gnet_stats_queue __percpu *cpu_qstats; int pad; refcount_t refcnt;
/* Cache line potentially dirtied in dequeue() or __netif_reschedule(). */ __cacheline_group_begin(Qdisc_read_mostly) ____cacheline_aligned; struct sk_buff_head gso_skb; struct Qdisc *next_sched; struct sk_buff_head skb_bad_txq; __cacheline_group_end(Qdisc_read_mostly);
/* Fields dirtied in dequeue() fast path. */ __cacheline_group_begin(Qdisc_write) ____cacheline_aligned; struct qdisc_skb_head q; unsigned long state; struct gnet_stats_basic_sync bstats; bool running; /* must be written under qdisc spinlock */
/* Note : we only change qstats.backlog in fast path. */ struct gnet_stats_queue qstats;
struct sk_buff *to_free; __cacheline_group_end(Qdisc_write);
atomic_long_t defer_count ____cacheline_aligned_in_smp; struct llist_head defer_list;
spinlock_t seqlock;
struct rcu_head rcu; netdevice_tracker dev_tracker; struct lock_class_key root_lock_key; /* private data */ long privdata[] ____cacheline_aligned;};
static inline void qdisc_refcount_inc(struct Qdisc *qdisc){ if (qdisc->flags & TCQ_F_BUILTIN) return; refcount_inc(&qdisc->refcnt);}
static inline bool qdisc_refcount_dec_if_one(struct Qdisc *qdisc){ if (qdisc->flags & TCQ_F_BUILTIN) return true; return refcount_dec_if_one(&qdisc->refcnt);}
/* Intended to be used by unlocked users, when concurrent qdisc release is * possible. */
static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc){ if (qdisc->flags & TCQ_F_BUILTIN) return qdisc; if (refcount_inc_not_zero(&qdisc->refcnt)) return qdisc; return NULL;}
/* For !TCQ_F_NOLOCK qdisc: callers must either call this within a qdisc * root_lock section, or provide their own memory barriers -- ordering * against qdisc_run_begin/end() atomic bit operations. */static inline bool qdisc_is_running(struct Qdisc *qdisc){ if (qdisc->flags & TCQ_F_NOLOCK) return spin_is_locked(&qdisc->seqlock); return READ_ONCE(qdisc->running);}
static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc){ return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY);}
static inline bool qdisc_is_percpu_stats(const struct Qdisc *q){ return q->flags & TCQ_F_CPUSTATS;}
static inline bool qdisc_is_empty(const struct Qdisc *qdisc){ if (qdisc_is_percpu_stats(qdisc)) return nolock_qdisc_is_empty(qdisc); return !READ_ONCE(qdisc->q.qlen);}
/* For !TCQ_F_NOLOCK qdisc, qdisc_run_begin/end() must be invoked with * the qdisc root lock acquired. */static inline bool qdisc_run_begin(struct Qdisc *qdisc){ if (qdisc->flags & TCQ_F_NOLOCK) { if (spin_trylock(&qdisc->seqlock)) return true;
/* No need to insist if the MISSED flag was already set. * Note that test_and_set_bit() also gives us memory ordering * guarantees wrt potential earlier enqueue() and below * spin_trylock(), both of which are necessary to prevent races */ if (test_and_set_bit(__QDISC_STATE_MISSED, &qdisc->state)) return false;
/* Try to take the lock again to make sure that we will either * grab it or the CPU that still has it will see MISSED set * when testing it in qdisc_run_end() */ return spin_trylock(&qdisc->seqlock); } if (READ_ONCE(qdisc->running)) return false; WRITE_ONCE(qdisc->running, true); return true;}
static inline struct sk_buff *qdisc_run_end(struct Qdisc *qdisc){ struct sk_buff *to_free = NULL;
if (qdisc->flags & TCQ_F_NOLOCK) { spin_unlock(&qdisc->seqlock);
/* spin_unlock() only has store-release semantic. The unlock * and test_bit() ordering is a store-load ordering, so a full * memory barrier is needed here. */ smp_mb();
if (unlikely(test_bit(__QDISC_STATE_MISSED, &qdisc->state))) __netif_schedule(qdisc); return NULL; }
if (qdisc->flags & TCQ_F_DEQUEUE_DROPS) { to_free = qdisc->to_free; if (to_free) qdisc->to_free = NULL; } WRITE_ONCE(qdisc->running, false); return to_free;}
static inline bool qdisc_may_bulk(const struct Qdisc *qdisc){ return qdisc->flags & TCQ_F_ONETXQUEUE;}
static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq){ return netdev_queue_dql_avail(txq);}
struct Qdisc_class_ops { unsigned int flags; /* Child qdisc manipulation */ struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); int (*graft)(struct Qdisc *, unsigned long cl, struct Qdisc *, struct Qdisc **, struct netlink_ext_ack *extack); struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); void (*qlen_notify)(struct Qdisc *, unsigned long);
/* Class manipulation routines */ unsigned long (*find)(struct Qdisc *, u32 classid); int (*change)(struct Qdisc *, u32, u32, struct nlattr **, unsigned long *, struct netlink_ext_ack *); int (*delete)(struct Qdisc *, unsigned long, struct netlink_ext_ack *); void (*walk)(struct Qdisc *, struct qdisc_walker * arg);
/* Filter manipulation */ struct tcf_block * (*tcf_block)(struct Qdisc *sch, unsigned long arg, struct netlink_ext_ack *extack); unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, u32 classid); void (*unbind_tcf)(struct Qdisc *, unsigned long);
/* rtnetlink specific */ int (*dump)(struct Qdisc *, unsigned long, struct sk_buff *skb, struct tcmsg*); int (*dump_stats)(struct Qdisc *, unsigned long, struct gnet_dump *);};
/* Qdisc_class_ops flag values */
/* Implements API that doesn't require rtnl lock */enum qdisc_class_ops_flags { QDISC_CLASS_OPS_DOIT_UNLOCKED = 1,};
struct Qdisc_ops { struct Qdisc_ops *next; const struct Qdisc_class_ops *cl_ops; char id[IFNAMSIZ]; int priv_size; unsigned int static_flags;
int (*enqueue)(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free); struct sk_buff * (*dequeue)(struct Qdisc *); struct sk_buff * (*peek)(struct Qdisc *);
int (*init)(struct Qdisc *sch, struct nlattr *arg, struct netlink_ext_ack *extack); void (*reset)(struct Qdisc *); void (*destroy)(struct Qdisc *); int (*change)(struct Qdisc *sch, struct nlattr *arg, struct netlink_ext_ack *extack); void (*attach)(struct Qdisc *sch); int (*change_tx_queue_len)(struct Qdisc *, unsigned int); void (*change_real_num_tx)(struct Qdisc *sch, unsigned int new_real_tx);
int (*dump)(struct Qdisc *, struct sk_buff *); int (*dump_stats)(struct Qdisc *, struct gnet_dump *);
void (*ingress_block_set)(struct Qdisc *sch, u32 block_index); void (*egress_block_set)(struct Qdisc *sch, u32 block_index); u32 (*ingress_block_get)(struct Qdisc *sch); u32 (*egress_block_get)(struct Qdisc *sch);
struct module *owner;};
struct tcf_result { union { struct { unsigned long class; u32 classid; }; const struct tcf_proto *goto_tp; };};
struct tcf_chain;
struct tcf_proto_ops { struct list_head head; char kind[IFNAMSIZ];
int (*classify)(struct sk_buff *, const struct tcf_proto *, struct tcf_result *); int (*init)(struct tcf_proto*); void (*destroy)(struct tcf_proto *tp, bool rtnl_held, struct netlink_ext_ack *extack);
void* (*get)(struct tcf_proto*, u32 handle); void (*put)(struct tcf_proto *tp, void *f); int (*change)(struct net *net, struct sk_buff *, struct tcf_proto*, unsigned long, u32 handle, struct nlattr **, void **, u32, struct netlink_ext_ack *); int (*delete)(struct tcf_proto *tp, void *arg, bool *last, bool rtnl_held, struct netlink_ext_ack *); bool (*delete_empty)(struct tcf_proto *tp); void (*walk)(struct tcf_proto *tp, struct tcf_walker *arg, bool rtnl_held); int (*reoffload)(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, void *cb_priv, struct netlink_ext_ack *extack); void (*hw_add)(struct tcf_proto *tp, void *type_data); void (*hw_del)(struct tcf_proto *tp, void *type_data); void (*bind_class)(void *, u32, unsigned long, void *, unsigned long); void * (*tmplt_create)(struct net *net, struct tcf_chain *chain, struct nlattr **tca, struct netlink_ext_ack *extack); void (*tmplt_destroy)(void *tmplt_priv); void (*tmplt_reoffload)(struct tcf_chain *chain, bool add, flow_setup_cb_t *cb, void *cb_priv); struct tcf_exts * (*get_exts)(const struct tcf_proto *tp, u32 handle);
/* rtnetlink specific */ int (*dump)(struct net*, struct tcf_proto*, void *, struct sk_buff *skb, struct tcmsg*, bool); int (*terse_dump)(struct net *net, struct tcf_proto *tp, void *fh, struct sk_buff *skb, struct tcmsg *t, bool rtnl_held); int (*tmplt_dump)(struct sk_buff *skb, struct net *net, void *tmplt_priv);
struct module *owner; int flags;};
/* Classifiers setting TCF_PROTO_OPS_DOIT_UNLOCKED in tcf_proto_ops->flags * are expected to implement tcf_proto_ops->delete_empty(), otherwise race * conditions can occur when filters are inserted/deleted simultaneously. */enum tcf_proto_ops_flags { TCF_PROTO_OPS_DOIT_UNLOCKED = 1,};
struct tcf_proto { /* Fast access part */ struct tcf_proto __rcu *next; void __rcu *root;
/* called under RCU BH lock*/ int (*classify)(struct sk_buff *, const struct tcf_proto *, struct tcf_result *); __be16 protocol;
/* All the rest */ u32 prio; void *data; const struct tcf_proto_ops *ops; struct tcf_chain *chain; /* Lock protects tcf_proto shared state and can be used by unlocked * classifiers to protect their private data. */ spinlock_t lock; bool deleting; bool counted; bool usesw; refcount_t refcnt; struct rcu_head rcu; struct hlist_node destroy_ht_node;};
struct qdisc_skb_cb { unsigned int pkt_len; u16 pkt_segs; u16 tc_classid;#define QDISC_CB_PRIV_LEN 20 unsigned char data[QDISC_CB_PRIV_LEN];
u16 slave_dev_queue_mapping; u8 post_ct:1; u8 post_ct_snat:1; u8 post_ct_dnat:1;};
typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
struct tcf_chain { /* Protects filter_chain. */ struct mutex filter_chain_lock; struct tcf_proto __rcu *filter_chain; struct list_head list; struct tcf_block *block; u32 index; /* chain index */ unsigned int refcnt; unsigned int action_refcnt; bool explicitly_created; bool flushing; const struct tcf_proto_ops *tmplt_ops; void *tmplt_priv; struct rcu_head rcu;};
struct tcf_block { struct xarray ports; /* datapath accessible */ /* Lock protects tcf_block and lifetime-management data of chains * attached to the block (refcnt, action_refcnt, explicitly_created). */ struct mutex lock; struct list_head chain_list; u32 index; /* block index for shared blocks */ u32 classid; /* which class this block belongs to */ refcount_t refcnt; struct net *net; struct Qdisc *q; struct rw_semaphore cb_lock; /* protects cb_list and offload counters */ struct flow_block flow_block; struct list_head owner_list; bool keep_dst; atomic_t useswcnt; atomic_t offloadcnt; /* Number of oddloaded filters */ unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */ unsigned int lockeddevcnt; /* Number of devs that require rtnl lock. */ struct { struct tcf_chain *chain; struct list_head filter_chain_list; } chain0; struct rcu_head rcu; DECLARE_HASHTABLE(proto_destroy_ht, 7); struct mutex proto_destroy_lock; /* Lock for proto_destroy hashtable. */};
struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index);
static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain){ return lockdep_is_held(&chain->filter_chain_lock);}
static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp){ return lockdep_is_held(&tp->lock);}
#define tcf_chain_dereference(p, chain) \ rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain))
#define tcf_proto_dereference(p, tp) \ rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp))
static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz){ struct qdisc_skb_cb *qcb;
BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*qcb)); BUILD_BUG_ON(sizeof(qcb->data) < sz);}
static inline int qdisc_qlen(const struct Qdisc *q){ return q->q.qlen;}
static inline int qdisc_qlen_lockless(const struct Qdisc *q){ return READ_ONCE(q->q.qlen);}
static inline void qdisc_qlen_inc(struct Qdisc *q){ WRITE_ONCE(q->q.qlen, q->q.qlen + 1);}
static inline void qdisc_qlen_dec(struct Qdisc *q){ WRITE_ONCE(q->q.qlen, q->q.qlen - 1);}
static inline int qdisc_qlen_sum(const struct Qdisc *q){ __u32 qlen = q->qstats.qlen; int i;
if (qdisc_is_percpu_stats(q)) { for_each_possible_cpu(i) qlen += READ_ONCE(per_cpu_ptr(q->cpu_qstats, i)->qlen); } else { qlen += qdisc_qlen_lockless(q); }
return qlen;}
static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb){ return (struct qdisc_skb_cb *)skb->cb;}
static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc){ return &qdisc->q.lock;}
static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc){ struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc);
return q;}
static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc){ return rcu_dereference_bh(qdisc->dev_queue->qdisc);}
static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc){ return rcu_dereference_rtnl(qdisc->dev_queue->qdisc_sleeping);}
static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc){ struct Qdisc *root = qdisc_root_sleeping(qdisc);
ASSERT_RTNL(); return qdisc_lock(root);}
static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc){ return qdisc->dev_queue->dev;}
static inline void sch_tree_lock(struct Qdisc *q){ if (q->flags & TCQ_F_MQROOT) spin_lock_bh(qdisc_lock(q)); else spin_lock_bh(qdisc_root_sleeping_lock(q));}
static inline void sch_tree_unlock(struct Qdisc *q){ if (q->flags & TCQ_F_MQROOT) spin_unlock_bh(qdisc_lock(q)); else spin_unlock_bh(qdisc_root_sleeping_lock(q));}
extern struct Qdisc noop_qdisc;extern struct Qdisc_ops noop_qdisc_ops;extern struct Qdisc_ops pfifo_fast_ops;extern const u8 sch_default_prio2band[TC_PRIO_MAX + 1];extern struct Qdisc_ops mq_qdisc_ops;extern struct Qdisc_ops noqueue_qdisc_ops;extern const struct Qdisc_ops *default_qdisc_ops;static inline const struct Qdisc_ops *get_default_qdisc_ops(const struct net_device *dev, int ntx){ return ntx < dev->real_num_tx_queues ? default_qdisc_ops : &pfifo_fast_ops;}
struct Qdisc_class_common { u32 classid; unsigned int filter_cnt; struct hlist_node hnode;};
struct Qdisc_class_hash { struct hlist_head *hash; unsigned int hashsize; unsigned int hashmask; unsigned int hashelems;};
static inline unsigned int qdisc_class_hash(u32 id, u32 mask){ id ^= id >> 8; id ^= id >> 4; return id & mask;}
static inline struct Qdisc_class_common *qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id){ struct Qdisc_class_common *cl; unsigned int h;
if (!id) return NULL;
h = qdisc_class_hash(id, hash->hashmask); hlist_for_each_entry(cl, &hash->hash[h], hnode) { if (cl->classid == id) return cl; } return NULL;}
static inline bool qdisc_class_in_use(const struct Qdisc_class_common *cl){ return cl->filter_cnt > 0;}
static inline void qdisc_class_get(struct Qdisc_class_common *cl){ unsigned int res;
if (check_add_overflow(cl->filter_cnt, 1, &res)) WARN(1, "Qdisc class overflow");
cl->filter_cnt = res;}
static inline void qdisc_class_put(struct Qdisc_class_common *cl){ unsigned int res;
if (check_sub_overflow(cl->filter_cnt, 1, &res)) WARN(1, "Qdisc class underflow");
cl->filter_cnt = res;}
static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid){ u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY;
return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL;}
int qdisc_class_hash_init(struct Qdisc_class_hash *);void qdisc_class_hash_insert(struct Qdisc_class_hash *, struct Qdisc_class_common *);void qdisc_class_hash_remove(struct Qdisc_class_hash *, struct Qdisc_class_common *);void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);void qdisc_class_hash_destroy(struct Qdisc_class_hash *);
int dev_qdisc_change_tx_queue_len(struct net_device *dev);void dev_qdisc_change_real_num_tx(struct net_device *dev, unsigned int new_real_tx);void dev_init_scheduler(struct net_device *dev);void dev_shutdown(struct net_device *dev);void dev_activate(struct net_device *dev);void dev_deactivate(struct net_device *dev, bool reset_needed);void dev_deactivate_many(struct list_head *head, bool reset_needed);struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, struct Qdisc *qdisc);void qdisc_reset(struct Qdisc *qdisc);void qdisc_destroy(struct Qdisc *qdisc);void qdisc_put(struct Qdisc *qdisc);void qdisc_put_unlocked(struct Qdisc *qdisc);void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
static inline void dev_reset_queue(struct net_device *dev, struct netdev_queue *dev_queue, void *_unused){ struct Qdisc *qdisc; bool nolock;
qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); if (!qdisc) return;
nolock = qdisc->flags & TCQ_F_NOLOCK;
if (nolock) spin_lock_bh(&qdisc->seqlock); spin_lock_bh(qdisc_lock(qdisc));
qdisc_reset(qdisc);
spin_unlock_bh(qdisc_lock(qdisc)); if (nolock) { clear_bit(__QDISC_STATE_MISSED, &qdisc->state); clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); spin_unlock_bh(&qdisc->seqlock); }}
#ifdef CONFIG_NET_SCHEDint qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, void *type_data);void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, struct Qdisc *new, struct Qdisc *old, enum tc_setup_type type, void *type_data, struct netlink_ext_ack *extack);#elsestatic inline intqdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, void *type_data){ q->flags &= ~TCQ_F_OFFLOADED; return 0;}
static inline voidqdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, struct Qdisc *new, struct Qdisc *old, enum tc_setup_type type, void *type_data, struct netlink_ext_ack *extack){}#endifvoid qdisc_offload_query_caps(struct net_device *dev, enum tc_setup_type type, void *caps, size_t caps_len);struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, struct netlink_ext_ack *extack);void qdisc_free_rcu(struct Qdisc *qdisc);struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, u32 parentid, struct netlink_ext_ack *extack);void __qdisc_calculate_pkt_len(struct sk_buff *skb, const struct qdisc_size_table *stab);int skb_do_redirect(struct sk_buff *);
static inline bool skb_at_tc_ingress(const struct sk_buff *skb){#ifdef CONFIG_NET_XGRESS return skb->tc_at_ingress;#else return false;#endif}
static inline bool skb_at_tc_egress(const struct sk_buff *skb){#ifdef CONFIG_NET_EGRESS return skb->nf_skip_egress && !skb_at_tc_ingress(skb);#else return false;#endif}
static inline bool skb_skip_tc_classify(struct sk_buff *skb){#ifdef CONFIG_NET_CLS_ACT if (skb->tc_skip_classify) { skb->tc_skip_classify = 0; return true; }#endif return false;}
/* Reset all TX qdiscs greater than index of a device. */static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i){ struct Qdisc *qdisc; bool nolock;
for (; i < dev->num_tx_queues; i++) { qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); if (qdisc) { nolock = qdisc->flags & TCQ_F_NOLOCK;
if (nolock) spin_lock_bh(&qdisc->seqlock); spin_lock_bh(qdisc_lock(qdisc)); qdisc_reset(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); if (nolock) { clear_bit(__QDISC_STATE_MISSED, &qdisc->state); clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); spin_unlock_bh(&qdisc->seqlock); } } }}
/* Are all TX queues of the device empty? */static inline bool qdisc_all_tx_empty(const struct net_device *dev){ unsigned int i;
rcu_read_lock(); for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); const struct Qdisc *q = rcu_dereference(txq->qdisc);
if (!qdisc_is_empty(q)) { rcu_read_unlock(); return false; } } rcu_read_unlock(); return true;}
/* Are any of the TX qdiscs changing? */static inline bool qdisc_tx_changing(const struct net_device *dev){ unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
if (rcu_access_pointer(txq->qdisc) != rcu_access_pointer(txq->qdisc_sleeping)) return true; } return false;}
/* "noqueue" qdisc identified by not having any enqueue, see noqueue_init() */static inline bool qdisc_txq_has_no_queue(const struct netdev_queue *txq){ struct Qdisc *qdisc = rcu_access_pointer(txq->qdisc);
return qdisc->enqueue == NULL;}
/* Is the device using the noop qdisc on all queues? */static inline bool qdisc_tx_is_noop(const struct net_device *dev){ unsigned int i;
for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) return false; } return true;}
static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb){ return qdisc_skb_cb(skb)->pkt_len;}
static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb){ u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;
DEBUG_NET_WARN_ON_ONCE(pkt_segs != (skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1)); return pkt_segs;}
/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */enum net_xmit_qdisc_t { __NET_XMIT_STOLEN = 0x00010000, __NET_XMIT_BYPASS = 0x00020000,};
#ifdef CONFIG_NET_CLS_ACT#define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1)#else#define net_xmit_drop_count(e) (1)#endif
static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, const struct Qdisc *sch){#ifdef CONFIG_NET_SCHED struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab);
if (stab) __qdisc_calculate_pkt_len(skb, stab);#endif}
static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free){ return sch->enqueue(skb, sch, to_free);}
static inline void _bstats_update(struct gnet_stats_basic_sync *bstats, __u64 bytes, __u64 packets){ u64_stats_update_begin(&bstats->syncp); u64_stats_add(&bstats->bytes, bytes); u64_stats_add(&bstats->packets, packets); u64_stats_update_end(&bstats->syncp);}
static inline void _bstats_set(struct gnet_stats_basic_sync *bstats, u64 bytes, u64 packets){ u64_stats_update_begin(&bstats->syncp); u64_stats_set(&bstats->bytes, bytes); u64_stats_set(&bstats->packets, packets); u64_stats_update_end(&bstats->syncp);}
static inline void bstats_update(struct gnet_stats_basic_sync *bstats, const struct sk_buff *skb){ _bstats_update(bstats, qdisc_pkt_len(skb), qdisc_pkt_segs(skb));}
static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, const struct sk_buff *skb){ bstats_update(this_cpu_ptr(sch->cpu_bstats), skb);}
static inline void qdisc_bstats_update(struct Qdisc *sch, const struct sk_buff *skb){ bstats_update(&sch->bstats, skb);}
static inline void qstats_backlog_sub(struct Qdisc *sch, u32 val){ WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog - val);}
static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, const struct sk_buff *skb){ qstats_backlog_sub(sch, qdisc_pkt_len(skb));}
static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, const struct sk_buff *skb){ this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));}
static inline void qstats_backlog_add(struct Qdisc *sch, u32 val){ WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog + val);}
static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, const struct sk_buff *skb){ qstats_backlog_add(sch, qdisc_pkt_len(skb));}
static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, const struct sk_buff *skb){ this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));}
static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch){ this_cpu_inc(sch->cpu_qstats->qlen);}
static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch){ this_cpu_dec(sch->cpu_qstats->qlen);}
static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch){ this_cpu_inc(sch->cpu_qstats->requeues);}
static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count){ WRITE_ONCE(sch->qstats.drops, sch->qstats.drops + count);}
static inline void qstats_drop_inc(struct gnet_stats_queue *qstats){ WRITE_ONCE(qstats->drops, qstats->drops + 1);}
static inline void qstats_cpu_drop_inc(struct gnet_stats_queue __percpu *qstats){ this_cpu_inc(qstats->drops);}
static inline void qstats_cpu_overlimit_inc(struct gnet_stats_queue __percpu *qstats){ this_cpu_inc(qstats->overlimits);}
static inline void qdisc_qstats_drop(struct Qdisc *sch){ qstats_drop_inc(&sch->qstats);}
static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch){ this_cpu_inc(sch->cpu_qstats->drops);}
static inline void qdisc_qstats_overlimit(struct Qdisc *sch){ WRITE_ONCE(sch->qstats.overlimits, sch->qstats.overlimits + 1);}
static inline int qdisc_qstats_copy(struct gnet_dump *d, const struct Qdisc *sch){ __u32 qlen = qdisc_qlen_sum(sch);
return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen);}
static inline void qdisc_qstats_qlen_backlog(const struct Qdisc *sch, u32 *qlen, u32 *backlog){ struct gnet_stats_queue qstats = { 0 };
gnet_stats_add_queue(&qstats, sch->cpu_qstats, &sch->qstats); *qlen = qstats.qlen + qdisc_qlen_lockless(sch); *backlog = qstats.backlog;}
static inline void qdisc_purge_queue(struct Qdisc *sch){ __u32 qlen, backlog;
qdisc_qstats_qlen_backlog(sch, &qlen, &backlog); qdisc_reset(sch); qdisc_tree_reduce_backlog(sch, qlen, backlog);}
static inline void __qdisc_enqueue_tail(struct sk_buff *skb, struct qdisc_skb_head *qh){ struct sk_buff *last = qh->tail;
if (last) { skb->next = NULL; last->next = skb; qh->tail = skb; } else { qh->tail = skb; qh->head = skb; } WRITE_ONCE(qh->qlen, qh->qlen + 1);}
static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch){ __qdisc_enqueue_tail(skb, &sch->q); qdisc_qstats_backlog_inc(sch, skb); return NET_XMIT_SUCCESS;}
static inline void __qdisc_enqueue_head(struct sk_buff *skb, struct qdisc_skb_head *qh){ skb->next = qh->head;
if (!qh->head) qh->tail = skb; qh->head = skb; WRITE_ONCE(qh->qlen, qh->qlen + 1);}
static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh){ struct sk_buff *skb = qh->head;
if (likely(skb != NULL)) { qh->head = skb->next; WRITE_ONCE(qh->qlen, qh->qlen - 1); if (qh->head == NULL) qh->tail = NULL; skb->next = NULL; }
return skb;}
static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool direct){ struct sk_buff *skb;
skb = __skb_dequeue(&sch->gso_skb); if (skb) { qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); return skb; } if (direct) { skb = __qdisc_dequeue_head(&sch->q); if (skb) qdisc_qstats_backlog_dec(sch, skb); return skb; } else { return sch->dequeue(sch); }}
static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch){ struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
if (likely(skb != NULL)) { qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); }
return skb;}
struct tc_skb_cb { struct qdisc_skb_cb qdisc_cb; u32 drop_reason;
u16 zone; /* Only valid if qdisc_skb_cb(skb)->post_ct = true */ u16 mru;};
static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb){ struct tc_skb_cb *cb = (struct tc_skb_cb *)skb->cb;
BUILD_BUG_ON(sizeof(*cb) > sizeof_field(struct sk_buff, cb)); return cb;}
/* TC classifier accessors - use enum skb_drop_reason */static inline enum skb_drop_reasontcf_get_drop_reason(const struct sk_buff *skb){ return (enum skb_drop_reason)tc_skb_cb(skb)->drop_reason;}
static inline void tcf_set_drop_reason(const struct sk_buff *skb, enum skb_drop_reason reason){ tc_skb_cb(skb)->drop_reason = (enum qdisc_drop_reason)reason;}
/* Qdisc accessors - use enum qdisc_drop_reason */static inline enum qdisc_drop_reasontcf_get_qdisc_drop_reason(const struct sk_buff *skb){ return tc_skb_cb(skb)->drop_reason;}
static inline void tcf_set_qdisc_drop_reason(const struct sk_buff *skb, enum qdisc_drop_reason reason){ tc_skb_cb(skb)->drop_reason = reason;}
void __tcf_kfree_skb_list(struct sk_buff *skb, struct Qdisc *q, struct netdev_queue *txq, struct net_device *dev);
static inline void tcf_kfree_skb_list(struct sk_buff *skb, struct Qdisc *q, struct netdev_queue *txq, struct net_device *dev){ if (unlikely(skb)) __tcf_kfree_skb_list(skb, q, txq, dev);}
static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb, enum qdisc_drop_reason reason){ struct Qdisc *root;
DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS)); DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK);
rcu_read_lock(); root = qdisc_root_sleeping(q);
if (root->flags & TCQ_F_DEQUEUE_DROPS) { tcf_set_qdisc_drop_reason(skb, reason); skb->next = root->to_free; root->to_free = skb; } else { kfree_skb_reason(skb, (enum skb_drop_reason)reason); } rcu_read_unlock();}
/* Instead of calling kfree_skb() while root qdisc lock is held, * queue the skb for future freeing at end of __dev_xmit_skb() */static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free){ skb->next = *to_free; *to_free = skb;}
static inline void __qdisc_drop_all(struct sk_buff *skb, struct sk_buff **to_free){ if (skb->prev) skb->prev->next = *to_free; else skb->next = *to_free; *to_free = skb;}
static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, struct qdisc_skb_head *qh, struct sk_buff **to_free){ struct sk_buff *skb = __qdisc_dequeue_head(qh);
if (likely(skb != NULL)) { unsigned int len = qdisc_pkt_len(skb);
qdisc_qstats_backlog_dec(sch, skb); __qdisc_drop(skb, to_free); return len; }
return 0;}
static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch){ const struct qdisc_skb_head *qh = &sch->q;
return qh->head;}
/* generic pseudo peek method for non-work-conserving qdisc */static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch){ struct sk_buff *skb = skb_peek(&sch->gso_skb);
/* we can reuse ->gso_skb because peek isn't called for root qdiscs */ if (!skb) { skb = sch->dequeue(sch);
if (skb) { __skb_queue_head(&sch->gso_skb, skb); /* it's still part of the queue */ qdisc_qstats_backlog_inc(sch, skb); qdisc_qlen_inc(sch); } }
return skb;}
static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, struct sk_buff *skb){ if (qdisc_is_percpu_stats(sch)) { qdisc_qstats_cpu_backlog_dec(sch, skb); qdisc_bstats_cpu_update(sch, skb); qdisc_qstats_cpu_qlen_dec(sch); } else { qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); qdisc_qlen_dec(sch); }}
static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, unsigned int pkt_len){ if (qdisc_is_percpu_stats(sch)) { qdisc_qstats_cpu_qlen_inc(sch); this_cpu_add(sch->cpu_qstats->backlog, pkt_len); } else { qstats_backlog_add(sch, pkt_len); qdisc_qlen_inc(sch); }}
/* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch){ struct sk_buff *skb = skb_peek(&sch->gso_skb);
if (skb) { skb = __skb_dequeue(&sch->gso_skb); if (qdisc_is_percpu_stats(sch)) { qdisc_qstats_cpu_backlog_dec(sch, skb); qdisc_qstats_cpu_qlen_dec(sch); } else { qdisc_qstats_backlog_dec(sch, skb); qdisc_qlen_dec(sch); } } else { skb = sch->dequeue(sch); }
return skb;}
static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh){ /* * We do not know the backlog in bytes of this list, it * is up to the caller to correct it */ ASSERT_RTNL(); if (qh->qlen) { rtnl_kfree_skbs(qh->head, qh->tail);
qh->head = NULL; qh->tail = NULL; WRITE_ONCE(qh->qlen, 0); }}
static inline void qdisc_reset_queue(struct Qdisc *sch){ __qdisc_reset_queue(&sch->q);}
static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, struct Qdisc **pold){ struct Qdisc *old;
sch_tree_lock(sch); old = *pold; *pold = new; if (old != NULL) qdisc_purge_queue(old); sch_tree_unlock(sch);
return old;}
static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch){ rtnl_kfree_skbs(skb, skb); qdisc_qstats_drop(sch);}
static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free){ __qdisc_drop(skb, to_free); qdisc_qstats_cpu_drop(sch);
return NET_XMIT_DROP;}
static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free){ __qdisc_drop(skb, to_free); qdisc_qstats_drop(sch);
return NET_XMIT_DROP;}
static inline int qdisc_drop_reason(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free, enum qdisc_drop_reason reason){ tcf_set_qdisc_drop_reason(skb, reason); return qdisc_drop(skb, sch, to_free);}
static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free){ __qdisc_drop_all(skb, to_free); qdisc_qstats_drop(sch);
return NET_XMIT_DROP;}
struct psched_ratecfg { u64 rate_bytes_ps; /* bytes per second */ u32 mult; u16 overhead; u16 mpu; u8 linklayer; u8 shift;};
static inline u64 psched_l2t_ns(const struct psched_ratecfg *r, unsigned int len){ len += r->overhead;
if (len < r->mpu) len = r->mpu;
if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift;
return ((u64)len * r->mult) >> r->shift;}
void psched_ratecfg_precompute(struct psched_ratecfg *r, const struct tc_ratespec *conf, u64 rate64);
static inline void psched_ratecfg_getrate(struct tc_ratespec *res, const struct psched_ratecfg *r){ memset(res, 0, sizeof(*res));
/* legacy struct tc_ratespec has a 32bit @rate field * Qdisc using 64bit rate should add new attributes * in order to maintain compatibility. */ res->rate = min_t(u64, r->rate_bytes_ps, ~0U);
res->overhead = r->overhead; res->mpu = r->mpu; res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);}
struct psched_pktrate { u64 rate_pkts_ps; /* packets per second */ u32 mult; u8 shift;};
static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r, unsigned int pkt_num){ return ((u64)pkt_num * r->mult) >> r->shift;}
void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64);
/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc. * The fast path only needs to access filter list and to update stats */struct mini_Qdisc { struct tcf_proto *filter_list; struct tcf_block *block; struct gnet_stats_basic_sync __percpu *cpu_bstats; struct gnet_stats_queue __percpu *cpu_qstats; unsigned long rcu_state;};
static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq, const struct sk_buff *skb){ bstats_update(this_cpu_ptr(miniq->cpu_bstats), skb);}
static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq){ this_cpu_inc(miniq->cpu_qstats->drops);}
struct mini_Qdisc_pair { struct mini_Qdisc miniq1; struct mini_Qdisc miniq2; struct mini_Qdisc __rcu **p_miniq;};
void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp, struct tcf_proto *tp_head);void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, struct mini_Qdisc __rcu **p_miniq);void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, struct tcf_block *block);
static inline bool mini_qdisc_pair_inited(struct mini_Qdisc_pair *miniqp){ return !!miniqp->p_miniq;}
void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx);
int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb));
/* Make sure qdisc is no longer in SCHED state. */static inline void qdisc_synchronize(const struct Qdisc *q){ while (test_bit(__QDISC_STATE_SCHED, &q->state)) msleep(1);}
#endif