Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414// SPDX-License-Identifier: GPL-2.0
#include "io_uring.h"#include "napi.h"
#ifdef CONFIG_NET_RX_BUSY_POLL
/* Timeout for cleanout of stale entries. */#define NAPI_TIMEOUT (60 * HZ)
struct io_napi_entry { unsigned int napi_id; struct list_head list;
unsigned long timeout; struct hlist_node node;
struct rcu_head rcu;};
static struct io_napi_entry *io_napi_hash_find(struct hlist_head *hash_list, unsigned int napi_id){ struct io_napi_entry *e;
hlist_for_each_entry_rcu(e, hash_list, node) { if (e->napi_id != napi_id) continue; return e; }
return NULL;}
static inline ktime_t net_to_ktime(unsigned long t){ /* napi approximating usecs, reverse busy_loop_current_time */ return ns_to_ktime(t << 10);}
int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id, unsigned int mode){ struct hlist_head *hash_list; struct io_napi_entry *e;
/* Non-NAPI IDs can be rejected. */ if (!napi_id_valid(napi_id)) return -EINVAL;
hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))];
scoped_guard(rcu) { e = io_napi_hash_find(hash_list, napi_id); if (e) { WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT); return -EEXIST; } }
e = kmalloc_obj(*e, GFP_NOWAIT); if (!e) return -ENOMEM;
e->napi_id = napi_id; e->timeout = jiffies + NAPI_TIMEOUT;
/* * guard(spinlock) is not used to manually unlock it before calling * kfree() */ spin_lock(&ctx->napi_lock); if (unlikely(READ_ONCE(ctx->napi_track_mode) != mode)) { spin_unlock(&ctx->napi_lock); kfree(e); return -EINVAL; } if (unlikely(io_napi_hash_find(hash_list, napi_id))) { spin_unlock(&ctx->napi_lock); kfree(e); return -EEXIST; }
hlist_add_tail_rcu(&e->node, hash_list); list_add_tail_rcu(&e->list, &ctx->napi_list); spin_unlock(&ctx->napi_lock); return 0;}
static int __io_napi_del_id(struct io_ring_ctx *ctx, unsigned int napi_id){ struct hlist_head *hash_list; struct io_napi_entry *e;
/* Non-NAPI IDs can be rejected. */ if (!napi_id_valid(napi_id)) return -EINVAL;
hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))]; guard(spinlock)(&ctx->napi_lock); e = io_napi_hash_find(hash_list, napi_id); if (!e) return -ENOENT;
list_del_rcu(&e->list); hash_del_rcu(&e->node); kfree_rcu(e, rcu); return 0;}
static void __io_napi_remove_stale(struct io_ring_ctx *ctx){ struct io_napi_entry *e;
guard(spinlock)(&ctx->napi_lock); /* * list_for_each_entry_safe() is not required as long as: * 1. list_del_rcu() does not reset the deleted node next pointer * 2. kfree_rcu() delays the memory freeing until the next quiescent * state */ list_for_each_entry(e, &ctx->napi_list, list) { if (time_after(jiffies, READ_ONCE(e->timeout))) { list_del_rcu(&e->list); hash_del_rcu(&e->node); kfree_rcu(e, rcu); } }}
static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale){ if (is_stale) __io_napi_remove_stale(ctx);}
static inline bool io_napi_busy_loop_timeout(ktime_t start_time, ktime_t bp){ if (bp) { ktime_t end_time = ktime_add(start_time, bp); ktime_t now = net_to_ktime(busy_loop_current_time());
return ktime_after(now, end_time); }
return true;}
static bool io_napi_busy_loop_should_end(void *data, unsigned long start_time){ struct io_wait_queue *iowq = data;
if (signal_pending(current)) return true; if (io_should_wake(iowq) || io_has_work(iowq->ctx)) return true; if (io_napi_busy_loop_timeout(net_to_ktime(start_time), iowq->napi_busy_poll_dt)) return true;
return false;}
/* * never report stale entries */static bool static_tracking_do_busy_loop(struct io_ring_ctx *ctx, bool (*loop_end)(void *, unsigned long), void *loop_end_arg){ struct io_napi_entry *e;
list_for_each_entry_rcu(e, &ctx->napi_list, list) napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET); return false;}
static booldynamic_tracking_do_busy_loop(struct io_ring_ctx *ctx, bool (*loop_end)(void *, unsigned long), void *loop_end_arg){ struct io_napi_entry *e; bool is_stale = false;
list_for_each_entry_rcu(e, &ctx->napi_list, list) { napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET);
if (time_after(jiffies, READ_ONCE(e->timeout))) is_stale = true; }
return is_stale;}
static inline bool__io_napi_do_busy_loop(struct io_ring_ctx *ctx, bool (*loop_end)(void *, unsigned long), void *loop_end_arg){ switch (READ_ONCE(ctx->napi_track_mode)) { case IO_URING_NAPI_TRACKING_STATIC: return static_tracking_do_busy_loop(ctx, loop_end, loop_end_arg); case IO_URING_NAPI_TRACKING_DYNAMIC: return dynamic_tracking_do_busy_loop(ctx, loop_end, loop_end_arg); default: return false; }}
static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq){ unsigned long start_time = busy_loop_current_time(); bool (*loop_end)(void *, unsigned long) = NULL; void *loop_end_arg = NULL; bool is_stale = false;
/* Singular lists use a different napi loop end check function and are * only executed once. */ if (list_is_singular(&ctx->napi_list)) { loop_end = io_napi_busy_loop_should_end; loop_end_arg = iowq; }
scoped_guard(rcu) { do { is_stale = __io_napi_do_busy_loop(ctx, loop_end, loop_end_arg); } while (!io_napi_busy_loop_should_end(iowq, start_time) && !loop_end_arg); }
io_napi_remove_stale(ctx, is_stale);}
/* * io_napi_init() - Init napi settings * @ctx: pointer to io-uring context structure * * Init napi settings in the io-uring context. */void io_napi_init(struct io_ring_ctx *ctx){ u64 sys_dt = READ_ONCE(sysctl_net_busy_poll) * NSEC_PER_USEC;
INIT_LIST_HEAD(&ctx->napi_list); spin_lock_init(&ctx->napi_lock); ctx->napi_prefer_busy_poll = false; ctx->napi_busy_poll_dt = ns_to_ktime(sys_dt); ctx->napi_track_mode = IO_URING_NAPI_TRACKING_INACTIVE;}
/* * io_napi_free() - Deallocate napi * @ctx: pointer to io-uring context structure * * Free the napi list and the hash table in the io-uring context. */void io_napi_free(struct io_ring_ctx *ctx){ struct io_napi_entry *e;
guard(spinlock)(&ctx->napi_lock); list_for_each_entry(e, &ctx->napi_list, list) { hash_del_rcu(&e->node); kfree_rcu(e, rcu); } INIT_LIST_HEAD_RCU(&ctx->napi_list);}
static int io_napi_register_napi(struct io_ring_ctx *ctx, struct io_uring_napi *napi){ switch (napi->op_param) { case IO_URING_NAPI_TRACKING_DYNAMIC: case IO_URING_NAPI_TRACKING_STATIC: break; default: return -EINVAL; } WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE); io_napi_free(ctx); /* cap NAPI at 10 msec of spin time */ napi->busy_poll_to = min(10000, napi->busy_poll_to); WRITE_ONCE(ctx->napi_busy_poll_dt, napi->busy_poll_to * NSEC_PER_USEC); WRITE_ONCE(ctx->napi_prefer_busy_poll, !!napi->prefer_busy_poll); WRITE_ONCE(ctx->napi_track_mode, napi->op_param); return 0;}
/* * io_napi_register() - Register napi with io-uring * @ctx: pointer to io-uring context structure * @arg: pointer to io_uring_napi structure * * Register napi in the io-uring context. */int io_register_napi(struct io_ring_ctx *ctx, void __user *arg){ const struct io_uring_napi curr = { .busy_poll_to = ktime_to_us(ctx->napi_busy_poll_dt), .prefer_busy_poll = ctx->napi_prefer_busy_poll, .op_param = ctx->napi_track_mode }; struct io_uring_napi napi;
if (ctx->flags & IORING_SETUP_IOPOLL) return -EINVAL; if (copy_from_user(&napi, arg, sizeof(napi))) return -EFAULT; if (napi.pad[0] || napi.pad[1] || napi.resv) return -EINVAL;
if (copy_to_user(arg, &curr, sizeof(curr))) return -EFAULT;
switch (napi.opcode) { case IO_URING_NAPI_REGISTER_OP: return io_napi_register_napi(ctx, &napi); case IO_URING_NAPI_STATIC_ADD_ID: if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC) return -EINVAL; return __io_napi_add_id(ctx, napi.op_param, IO_URING_NAPI_TRACKING_STATIC); case IO_URING_NAPI_STATIC_DEL_ID: if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC) return -EINVAL; return __io_napi_del_id(ctx, napi.op_param); default: return -EINVAL; }}
/* * io_napi_unregister() - Unregister napi with io-uring * @ctx: pointer to io-uring context structure * @arg: pointer to io_uring_napi structure * * Unregister napi. If arg has been specified copy the busy poll timeout and * prefer busy poll setting to the passed in structure. */int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg){ const struct io_uring_napi curr = { .busy_poll_to = ktime_to_us(ctx->napi_busy_poll_dt), .prefer_busy_poll = ctx->napi_prefer_busy_poll };
if (arg && copy_to_user(arg, &curr, sizeof(curr))) return -EFAULT;
WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE); WRITE_ONCE(ctx->napi_busy_poll_dt, 0); WRITE_ONCE(ctx->napi_prefer_busy_poll, false); io_napi_free(ctx); return 0;}
/* * __io_napi_busy_loop() - execute busy poll loop * @ctx: pointer to io-uring context structure * @iowq: pointer to io wait queue * * Execute the busy poll loop and merge the spliced off list. */void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq){ if (ctx->flags & IORING_SETUP_SQPOLL) return;
iowq->napi_busy_poll_dt = READ_ONCE(ctx->napi_busy_poll_dt); if (iowq->timeout != KTIME_MAX) { ktime_t dt = ktime_sub(iowq->timeout, io_get_time(ctx));
/* an absolute timeout may have passed already */ dt = max_t(ktime_t, dt, 0); iowq->napi_busy_poll_dt = min_t(u64, iowq->napi_busy_poll_dt, dt); }
iowq->napi_prefer_busy_poll = READ_ONCE(ctx->napi_prefer_busy_poll); io_napi_blocking_busy_loop(ctx, iowq);}
/* * io_napi_sqpoll_busy_poll() - busy poll loop for sqpoll * @ctx: pointer to io-uring context structure * * Splice of the napi list and execute the napi busy poll loop. */int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx){ bool is_stale = false;
if (!READ_ONCE(ctx->napi_busy_poll_dt)) return 0; if (list_empty_careful(&ctx->napi_list)) return 0;
scoped_guard(rcu) { is_stale = __io_napi_do_busy_loop(ctx, NULL, NULL); }
io_napi_remove_stale(ctx, is_stale); return 1;}
#endif