Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459// SPDX-License-Identifier: GPL-2.0/* * BPF filter support for io_uring. Supports SQE opcodes for now. */#include <linux/kernel.h>#include <linux/errno.h>#include <linux/io_uring.h>#include <linux/filter.h>#include <linux/bpf.h>#include <uapi/linux/io_uring.h>
#include "io_uring.h"#include "bpf_filter.h"#include "net.h"#include "openclose.h"
struct io_bpf_filter { refcount_t refs; struct bpf_prog *prog; struct io_bpf_filter *next;};
/* Deny if this is set as the filter */static const struct io_bpf_filter dummy_filter;
static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req){ const struct io_issue_def *def = &io_issue_defs[req->opcode];
bctx->opcode = req->opcode; bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS; bctx->user_data = req->cqe.user_data; /* clear residual, anything from pdu_size and below */ memset((void *) bctx + offsetof(struct io_uring_bpf_ctx, pdu_size), 0, sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));
/* * Opcodes can provide a handler for populating more data into bctx, * for filters to use. */ if (def->filter_pdu_size) { bctx->pdu_size = def->filter_pdu_size; def->filter_populate(bctx, req); }}
/* * Run registered filters for a given opcode. For filters, a return of 0 denies * execution of the request, a return of 1 allows it. If any filter for an * opcode returns 0, filter processing is stopped, and the request is denied. * This also stops the processing of filters. * * __io_uring_run_bpf_filters() returns 0 on success, allow running the * request, and -EACCES when a request is denied. */int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters, struct io_kiocb *req){ struct io_bpf_filter *filter; struct io_uring_bpf_ctx bpf_ctx; int ret;
/* Fast check for existence of filters outside of RCU */ if (!rcu_access_pointer(filters[req->opcode])) return 0;
/* * req->opcode has already been validated to be within the range * of what we expect, io_init_req() does this. */ guard(rcu)(); filter = rcu_dereference(filters[req->opcode]); if (!filter) return 0; else if (filter == &dummy_filter) return -EACCES;
io_uring_populate_bpf_ctx(&bpf_ctx, req);
/* * Iterate registered filters. The opcode is allowed IFF all filters * return 1. If any filter returns denied, opcode will be denied. */ do { if (filter == &dummy_filter) return -EACCES; ret = bpf_prog_run_pin_on_cpu(filter->prog, &bpf_ctx); if (!ret) return -EACCES; filter = filter->next; } while (filter);
return 0;}
static void io_free_bpf_filters(struct rcu_head *head){ struct io_bpf_filter __rcu **filter; struct io_bpf_filters *filters; int i;
filters = container_of(head, struct io_bpf_filters, rcu_head); scoped_guard(spinlock, &filters->lock) { filter = filters->filters; if (!filter) return; }
for (i = 0; i < IORING_OP_LAST; i++) { struct io_bpf_filter *f;
rcu_read_lock(); f = rcu_dereference(filter[i]); while (f) { struct io_bpf_filter *next = f->next;
/* * Even if stacked, dummy filter will always be last * as it can only get installed into an empty spot. */ if (f == &dummy_filter) break;
/* Someone still holds a ref, stop iterating. */ if (!refcount_dec_and_test(&f->refs)) break;
bpf_prog_destroy(f->prog); kfree(f); f = next; } rcu_read_unlock(); } kfree(filters->filters); kfree(filters);}
static void __io_put_bpf_filters(struct io_bpf_filters *filters){ if (refcount_dec_and_test(&filters->refs)) call_rcu(&filters->rcu_head, io_free_bpf_filters);}
void io_put_bpf_filters(struct io_restriction *res){ if (res->bpf_filters) __io_put_bpf_filters(res->bpf_filters);}
static struct io_bpf_filters *io_new_bpf_filters(void){ struct io_bpf_filters *filters __free(kfree) = NULL;
filters = kzalloc_obj(*filters, GFP_KERNEL_ACCOUNT); if (!filters) return ERR_PTR(-ENOMEM);
filters->filters = kzalloc_objs(struct io_bpf_filter *, IORING_OP_LAST, GFP_KERNEL_ACCOUNT); if (!filters->filters) return ERR_PTR(-ENOMEM);
refcount_set(&filters->refs, 1); spin_lock_init(&filters->lock); return no_free_ptr(filters);}
/* * Validate classic BPF filter instructions. Only allow a safe subset of * operations - no packet data access, just context field loads and basic * ALU/jump operations. */static int io_uring_check_cbpf_filter(struct sock_filter *filter, unsigned int flen){ int pc;
for (pc = 0; pc < flen; pc++) { struct sock_filter *ftest = &filter[pc]; u16 code = ftest->code; u32 k = ftest->k;
switch (code) { case BPF_LD | BPF_W | BPF_ABS: ftest->code = BPF_LDX | BPF_W | BPF_ABS; /* 32-bit aligned and not out of bounds. */ if (k >= sizeof(struct io_uring_bpf_ctx) || k & 3) return -EINVAL; continue; case BPF_LD | BPF_W | BPF_LEN: ftest->code = BPF_LD | BPF_IMM; ftest->k = sizeof(struct io_uring_bpf_ctx); continue; case BPF_LDX | BPF_W | BPF_LEN: ftest->code = BPF_LDX | BPF_IMM; ftest->k = sizeof(struct io_uring_bpf_ctx); continue; /* Explicitly include allowed calls. */ case BPF_RET | BPF_K: case BPF_RET | BPF_A: case BPF_ALU | BPF_ADD | BPF_K: case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU | BPF_AND | BPF_X: case BPF_ALU | BPF_OR | BPF_K: case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU | BPF_LSH | BPF_X: case BPF_ALU | BPF_RSH | BPF_K: case BPF_ALU | BPF_RSH | BPF_X: case BPF_ALU | BPF_NEG: case BPF_LD | BPF_IMM: case BPF_LDX | BPF_IMM: case BPF_MISC | BPF_TAX: case BPF_MISC | BPF_TXA: case BPF_LD | BPF_MEM: case BPF_LDX | BPF_MEM: case BPF_ST: case BPF_STX: case BPF_JMP | BPF_JA: case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP | BPF_JGE | BPF_K: case BPF_JMP | BPF_JGE | BPF_X: case BPF_JMP | BPF_JGT | BPF_K: case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP | BPF_JSET | BPF_K: case BPF_JMP | BPF_JSET | BPF_X: continue; default: return -EINVAL; } } return 0;}
void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src){ if (!src->bpf_filters) return;
rcu_read_lock(); /* * If the src filter is going away, just ignore it. */ if (refcount_inc_not_zero(&src->bpf_filters->refs)) { src->bpf_filters_cow = true; dst->bpf_filters = src->bpf_filters; dst->bpf_filters_cow = true; } rcu_read_unlock();}
/* * Allocate a new struct io_bpf_filters. Used when a filter is cloned and * modifications need to be made. */static struct io_bpf_filters *io_bpf_filter_cow(struct io_restriction *src){ struct io_bpf_filters *filters; struct io_bpf_filter *srcf; int i;
filters = io_new_bpf_filters(); if (IS_ERR(filters)) return filters;
/* * Iterate filters from src and assign in destination. Grabbing * a reference is enough, we don't need to duplicate the memory. * This is safe because filters are only ever appended to the * front of the list, hence the only memory ever touched inside * a filter is the refcount. */ rcu_read_lock(); for (i = 0; i < IORING_OP_LAST; i++) { srcf = rcu_dereference(src->bpf_filters->filters[i]); if (!srcf) { continue; } else if (srcf == &dummy_filter) { rcu_assign_pointer(filters->filters[i], &dummy_filter); continue; }
/* * Getting a ref on the first node is enough, putting the * filter and iterating nodes to free will stop on the first * one that doesn't hit zero when dropping. */ if (!refcount_inc_not_zero(&srcf->refs)) goto err; rcu_assign_pointer(filters->filters[i], srcf); } rcu_read_unlock(); return filters;err: rcu_read_unlock(); __io_put_bpf_filters(filters); return ERR_PTR(-EBUSY);}
#define IO_URING_BPF_FILTER_FLAGS (IO_URING_BPF_FILTER_DENY_REST | \ IO_URING_BPF_FILTER_SZ_STRICT)
static int io_bpf_filter_import(struct io_uring_bpf *reg, struct io_uring_bpf __user *arg){ const struct io_issue_def *def; int ret;
if (copy_from_user(reg, arg, sizeof(*reg))) return -EFAULT; if (reg->cmd_type != IO_URING_BPF_CMD_FILTER) return -EINVAL; if (reg->cmd_flags || reg->resv) return -EINVAL;
if (reg->filter.opcode >= IORING_OP_LAST) return -EINVAL; if (reg->filter.flags & ~IO_URING_BPF_FILTER_FLAGS) return -EINVAL; if (!mem_is_zero(reg->filter.resv, sizeof(reg->filter.resv))) return -EINVAL; if (!mem_is_zero(reg->filter.resv2, sizeof(reg->filter.resv2))) return -EINVAL; if (!reg->filter.filter_len || reg->filter.filter_len > BPF_MAXINSNS) return -EINVAL;
/* Verify filter size */ def = &io_issue_defs[array_index_nospec(reg->filter.opcode, IORING_OP_LAST)];
/* same size, always ok */ ret = 0; if (reg->filter.pdu_size == def->filter_pdu_size) ; /* size differs, fail in strict mode */ else if (reg->filter.flags & IO_URING_BPF_FILTER_SZ_STRICT) ret = -EMSGSIZE; /* userspace filter is bigger, always disallow */ else if (reg->filter.pdu_size > def->filter_pdu_size) ret = -EMSGSIZE;
/* copy back kernel filter size */ reg->filter.pdu_size = def->filter_pdu_size; if (copy_to_user(&arg->filter, ®->filter, sizeof(reg->filter))) return -EFAULT;
return ret;}
int io_register_bpf_filter(struct io_restriction *res, struct io_uring_bpf __user *arg){ struct io_bpf_filters *filters, *old_filters = NULL; struct io_bpf_filter *filter, *old_filter; struct io_uring_bpf reg; struct bpf_prog *prog; struct sock_fprog fprog; int ret;
ret = io_bpf_filter_import(®, arg); if (ret) return ret;
fprog.len = reg.filter.filter_len; fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);
ret = bpf_prog_create_from_user(&prog, &fprog, io_uring_check_cbpf_filter, false); if (ret) return ret;
/* * No existing filters, allocate set. */ filters = res->bpf_filters; if (!filters) { filters = io_new_bpf_filters(); if (IS_ERR(filters)) { ret = PTR_ERR(filters); goto err_prog; } } else if (res->bpf_filters_cow) { filters = io_bpf_filter_cow(res); if (IS_ERR(filters)) { ret = PTR_ERR(filters); goto err_prog; } /* * Stash old filters, we'll put them once we know we'll * succeed. Until then, res->bpf_filters is left untouched. */ old_filters = res->bpf_filters; }
filter = kzalloc_obj(*filter, GFP_KERNEL_ACCOUNT); if (!filter) { ret = -ENOMEM; goto err; } refcount_set(&filter->refs, 1); filter->prog = prog;
/* * Success - install the new filter set now. If we did COW, put * the old filters as we're replacing them. */ if (old_filters) { __io_put_bpf_filters(old_filters); res->bpf_filters_cow = false; } res->bpf_filters = filters;
/* * Insert filter - if the current opcode already has a filter * attached, add to the set. */ rcu_read_lock(); spin_lock_bh(&filters->lock); old_filter = rcu_dereference(filters->filters[reg.filter.opcode]); if (old_filter) filter->next = old_filter; rcu_assign_pointer(filters->filters[reg.filter.opcode], filter);
/* * If IO_URING_BPF_FILTER_DENY_REST is set, fill any unregistered * opcode with the dummy filter. That will cause them to be denied. */ if (reg.filter.flags & IO_URING_BPF_FILTER_DENY_REST) { for (int i = 0; i < IORING_OP_LAST; i++) { if (i == reg.filter.opcode) continue; old_filter = rcu_dereference(filters->filters[i]); if (old_filter) continue; rcu_assign_pointer(filters->filters[i], &dummy_filter); } }
spin_unlock_bh(&filters->lock); rcu_read_unlock(); return 0;err: if (filters != res->bpf_filters) __io_put_bpf_filters(filters);err_prog: bpf_prog_destroy(prog); return ret;}