Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390// SPDX-License-Identifier: GPL-2.0/* * A scheduler that validates ops.dequeue() is called correctly: * - Tasks dispatched to terminal DSQs (local, global) bypass the BPF * scheduler entirely: no ops.dequeue() should be called * - Tasks dispatched to user DSQs from ops.enqueue() enter BPF custody: * ops.dequeue() must be called when they leave custody * - Every ops.enqueue() dispatch to non-terminal DSQs is followed by * exactly one ops.dequeue() (validate 1:1 pairing and state machine) * * Copyright (c) 2026 NVIDIA Corporation. */
#include <scx/common.bpf.h>
#define SHARED_DSQ 0
/* * BPF internal queue. * * Tasks are stored here and consumed from ops.dispatch(), validating that * tasks on BPF internal structures still get ops.dequeue() when they * leave. */struct { __uint(type, BPF_MAP_TYPE_QUEUE); __uint(max_entries, 32768); __type(value, s32);} global_queue SEC(".maps");
char _license[] SEC("license") = "GPL";
UEI_DEFINE(uei);
/* * Counters to track the lifecycle of tasks: * - enqueue_cnt: Number of times ops.enqueue() was called * - dequeue_cnt: Number of times ops.dequeue() was called (any type) * - dispatch_dequeue_cnt: Number of regular dispatch dequeues (no flag) * - change_dequeue_cnt: Number of property change dequeues * - bpf_queue_full: Number of times the BPF internal queue was full */u64 enqueue_cnt, dequeue_cnt, dispatch_dequeue_cnt, change_dequeue_cnt, bpf_queue_full;
/* * Test scenarios: * 0) Dispatch to local DSQ from ops.select_cpu() (terminal DSQ, bypasses BPF * scheduler, no dequeue callbacks) * 1) Dispatch to global DSQ from ops.select_cpu() (terminal DSQ, bypasses BPF * scheduler, no dequeue callbacks) * 2) Dispatch to shared user DSQ from ops.select_cpu() (enters BPF scheduler, * dequeue callbacks expected) * 3) Dispatch to local DSQ from ops.enqueue() (terminal DSQ, bypasses BPF * scheduler, no dequeue callbacks) * 4) Dispatch to global DSQ from ops.enqueue() (terminal DSQ, bypasses BPF * scheduler, no dequeue callbacks) * 5) Dispatch to shared user DSQ from ops.enqueue() (enters BPF scheduler, * dequeue callbacks expected) * 6) BPF internal queue from ops.enqueue(): store task PIDs in ops.enqueue(), * consume in ops.dispatch() and dispatch to local DSQ (validates dequeue * for tasks stored in internal BPF data structures) */u32 test_scenario;
/* * Per-task state to track lifecycle and validate workflow semantics. * State transitions: * NONE -> ENQUEUED (on enqueue) * NONE -> DISPATCHED (on direct dispatch to terminal DSQ) * ENQUEUED -> DISPATCHED (on dispatch dequeue) * DISPATCHED -> NONE (on property change dequeue or re-enqueue) * ENQUEUED -> NONE (on property change dequeue before dispatch) */enum task_state { TASK_NONE = 0, TASK_ENQUEUED, TASK_DISPATCHED,};
struct task_ctx { enum task_state state; /* Current state in the workflow */ u64 enqueue_seq; /* Sequence number for debugging */};
struct { __uint(type, BPF_MAP_TYPE_TASK_STORAGE); __uint(map_flags, BPF_F_NO_PREALLOC); __type(key, int); __type(value, struct task_ctx);} task_ctx_stor SEC(".maps");
static struct task_ctx *try_lookup_task_ctx(struct task_struct *p){ return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);}
s32 BPF_STRUCT_OPS(dequeue_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags){ struct task_ctx *tctx;
tctx = try_lookup_task_ctx(p); if (!tctx) return prev_cpu;
switch (test_scenario) { case 0: /* * Direct dispatch to the local DSQ. * * Task bypasses BPF scheduler entirely: no enqueue * tracking, no ops.dequeue() callbacks. */ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0); tctx->state = TASK_DISPATCHED; break; case 1: /* * Direct dispatch to the global DSQ. * * Task bypasses BPF scheduler entirely: no enqueue * tracking, no ops.dequeue() callbacks. */ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0); tctx->state = TASK_DISPATCHED; break; case 2: /* * Dispatch to a shared user DSQ. * * Task enters BPF scheduler management: track * enqueue/dequeue lifecycle and validate state * transitions. */ if (tctx->state == TASK_ENQUEUED) scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu", p->pid, p->comm, tctx->enqueue_seq);
scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, 0);
__sync_fetch_and_add(&enqueue_cnt, 1);
tctx->state = TASK_ENQUEUED; tctx->enqueue_seq++; break; }
return prev_cpu;}
void BPF_STRUCT_OPS(dequeue_enqueue, struct task_struct *p, u64 enq_flags){ struct task_ctx *tctx; s32 pid = p->pid;
tctx = try_lookup_task_ctx(p); if (!tctx) return;
switch (test_scenario) { case 3: /* * Direct dispatch to the local DSQ. * * Task bypasses BPF scheduler entirely: no enqueue * tracking, no ops.dequeue() callbacks. */ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags); tctx->state = TASK_DISPATCHED; break; case 4: /* * Direct dispatch to the global DSQ. * * Task bypasses BPF scheduler entirely: no enqueue * tracking, no ops.dequeue() callbacks. */ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); tctx->state = TASK_DISPATCHED; break; case 5: /* * Dispatch to shared user DSQ. * * Task enters BPF scheduler management: track * enqueue/dequeue lifecycle and validate state * transitions. */ if (tctx->state == TASK_ENQUEUED) scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu", p->pid, p->comm, tctx->enqueue_seq);
scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags);
__sync_fetch_and_add(&enqueue_cnt, 1);
tctx->state = TASK_ENQUEUED; tctx->enqueue_seq++; break; case 6: /* * Store task in BPF internal queue. * * Task enters BPF scheduler management: track * enqueue/dequeue lifecycle and validate state * transitions. */ if (tctx->state == TASK_ENQUEUED) scx_bpf_error("%d (%s): enqueue while in ENQUEUED state seq=%llu", p->pid, p->comm, tctx->enqueue_seq);
if (bpf_map_push_elem(&global_queue, &pid, 0)) { scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); __sync_fetch_and_add(&bpf_queue_full, 1);
tctx->state = TASK_DISPATCHED; } else { __sync_fetch_and_add(&enqueue_cnt, 1);
tctx->state = TASK_ENQUEUED; tctx->enqueue_seq++; } break; default: /* For all other scenarios, dispatch to the global DSQ */ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); tctx->state = TASK_DISPATCHED; break; }
scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE);}
void BPF_STRUCT_OPS(dequeue_dequeue, struct task_struct *p, u64 deq_flags){ struct task_ctx *tctx;
__sync_fetch_and_add(&dequeue_cnt, 1);
tctx = try_lookup_task_ctx(p); if (!tctx) return;
/* * For scenarios 0, 1, 3, and 4 (terminal DSQs: local and global), * ops.dequeue() should never be called because tasks bypass the * BPF scheduler entirely. If we get here, it's a kernel bug. */ if (test_scenario == 0 || test_scenario == 3) { scx_bpf_error("%d (%s): dequeue called for local DSQ scenario", p->pid, p->comm); return; }
if (test_scenario == 1 || test_scenario == 4) { scx_bpf_error("%d (%s): dequeue called for global DSQ scenario", p->pid, p->comm); return; }
if (deq_flags & SCX_DEQ_SCHED_CHANGE) { /* * Property change interrupting the workflow. Valid from * both ENQUEUED and DISPATCHED states. Transitions task * back to NONE state. */ __sync_fetch_and_add(&change_dequeue_cnt, 1);
/* Validate state transition */ if (tctx->state != TASK_ENQUEUED && tctx->state != TASK_DISPATCHED) scx_bpf_error("%d (%s): invalid property change dequeue state=%d seq=%llu", p->pid, p->comm, tctx->state, tctx->enqueue_seq);
/* * Transition back to NONE: task outside scheduler control. * * Scenario 6: dispatch() checks tctx->state after popping a * PID, if the task is in state NONE, it was dequeued by * property change and must not be dispatched (this * prevents "target CPU not allowed"). */ tctx->state = TASK_NONE; } else { /* * Regular dispatch dequeue: kernel is moving the task from * BPF custody to a terminal DSQ. Normally we come from * ENQUEUED state. We can also see TASK_NONE if the task * was dequeued by property change (SCX_DEQ_SCHED_CHANGE) * while it was already on a DSQ (dispatched but not yet * consumed); in that case we just leave state as NONE. */ __sync_fetch_and_add(&dispatch_dequeue_cnt, 1);
/* * Must be ENQUEUED (normal path) or NONE (already dequeued * by property change while on a DSQ). */ if (tctx->state != TASK_ENQUEUED && tctx->state != TASK_NONE) scx_bpf_error("%d (%s): dispatch dequeue from state %d seq=%llu", p->pid, p->comm, tctx->state, tctx->enqueue_seq);
if (tctx->state == TASK_ENQUEUED) tctx->state = TASK_DISPATCHED;
/* NONE: leave as-is, task was already property-change dequeued */ }}
void BPF_STRUCT_OPS(dequeue_dispatch, s32 cpu, struct task_struct *prev){ if (test_scenario == 6) { struct task_ctx *tctx; struct task_struct *p; s32 pid;
if (bpf_map_pop_elem(&global_queue, &pid)) return;
p = bpf_task_from_pid(pid); if (!p) return;
/* * If the task was dequeued by property change * (ops.dequeue() set tctx->state = TASK_NONE), skip * dispatch. */ tctx = try_lookup_task_ctx(p); if (!tctx || tctx->state == TASK_NONE) { bpf_task_release(p); return; }
/* * Dispatch to this CPU's local DSQ if allowed, otherwise * fallback to the global DSQ. */ if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_DFL, 0); else scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
bpf_task_release(p); } else { scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); }}
s32 BPF_STRUCT_OPS(dequeue_init_task, struct task_struct *p, struct scx_init_task_args *args){ struct task_ctx *tctx;
tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); if (!tctx) return -ENOMEM;
return 0;}
s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_init){ s32 ret;
ret = scx_bpf_create_dsq(SHARED_DSQ, -1); if (ret) return ret;
return 0;}
void BPF_STRUCT_OPS(dequeue_exit, struct scx_exit_info *ei){ UEI_RECORD(uei, ei);}
SEC(".struct_ops.link")struct sched_ext_ops dequeue_ops = { .select_cpu = (void *)dequeue_select_cpu, .enqueue = (void *)dequeue_enqueue, .dequeue = (void *)dequeue_dequeue, .dispatch = (void *)dequeue_dispatch, .init_task = (void *)dequeue_init_task, .init = (void *)dequeue_init, .exit = (void *)dequeue_exit, .flags = SCX_OPS_ENQ_LAST, .name = "dequeue_test",};