Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354// SPDX-License-Identifier: LGPL-2.1#define _GNU_SOURCE#include <assert.h>#include <pthread.h>#include <sched.h>#include <stdint.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <stddef.h>
#include "kselftest.h"#include "rseq.h"
#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID# define RSEQ_PERCPU RSEQ_PERCPU_MM_CIDstaticint get_current_cpu_id(void){ return rseq_current_mm_cid();}staticbool rseq_validate_cpu_id(void){ return rseq_mm_cid_available();}staticbool rseq_use_cpu_index(void){ return false; /* Use mm_cid */}#else# define RSEQ_PERCPU RSEQ_PERCPU_CPU_IDstaticint get_current_cpu_id(void){ return rseq_cpu_start();}staticbool rseq_validate_cpu_id(void){ return rseq_current_cpu_raw() >= 0;}staticbool rseq_use_cpu_index(void){ return true; /* Use cpu_id as index. */}#endif
struct percpu_lock_entry { intptr_t v;} __attribute__((aligned(128)));
struct percpu_lock { struct percpu_lock_entry c[CPU_SETSIZE];};
struct test_data_entry { intptr_t count;} __attribute__((aligned(128)));
struct spinlock_test_data { struct percpu_lock lock; struct test_data_entry c[CPU_SETSIZE]; int reps;};
struct percpu_list_node { intptr_t data; struct percpu_list_node *next;};
struct percpu_list_entry { struct percpu_list_node *head;} __attribute__((aligned(128)));
struct percpu_list { struct percpu_list_entry c[CPU_SETSIZE];};
/* A simple percpu spinlock. Returns the cpu lock was acquired on. */int rseq_this_cpu_lock(struct percpu_lock *lock){ int cpu;
for (;;) { int ret;
cpu = get_current_cpu_id(); ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ } /* * Acquire semantic when taking lock after control dependency. * Matches rseq_smp_store_release(). */ rseq_smp_acquire__after_ctrl_dep(); return cpu;}
void rseq_percpu_unlock(struct percpu_lock *lock, int cpu){ assert(lock->c[cpu].v == 1); /* * Release lock, with release semantic. Matches * rseq_smp_acquire__after_ctrl_dep(). */ rseq_smp_store_release(&lock->c[cpu].v, 0);}
void *test_percpu_spinlock_thread(void *arg){ struct spinlock_test_data *data = arg; int i, cpu;
if (rseq_register_current_thread()) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); } for (i = 0; i < data->reps; i++) { cpu = rseq_this_cpu_lock(&data->lock); data->c[cpu].count++; rseq_percpu_unlock(&data->lock, cpu); } if (rseq_unregister_current_thread()) { fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); }
return NULL;}
/* * A simple test which implements a sharded counter using a per-cpu * lock. Obviously real applications might prefer to simply use a * per-cpu increment; however, this is reasonable for a test and the * lock can be extended to synchronize more complicated operations. */void test_percpu_spinlock(void){ const int num_threads = 200; int i; uint64_t sum; pthread_t test_threads[num_threads]; struct spinlock_test_data data;
memset(&data, 0, sizeof(data)); data.reps = 5000;
for (i = 0; i < num_threads; i++) pthread_create(&test_threads[i], NULL, test_percpu_spinlock_thread, &data);
for (i = 0; i < num_threads; i++) pthread_join(test_threads[i], NULL);
sum = 0; for (i = 0; i < CPU_SETSIZE; i++) sum += data.c[i].count;
assert(sum == (uint64_t)data.reps * num_threads);}
void this_cpu_list_push(struct percpu_list *list, struct percpu_list_node *node, int *_cpu){ int cpu;
for (;;) { intptr_t *targetptr, newval, expect; int ret;
cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ } if (_cpu) *_cpu = cpu;}
/* * Unlike a traditional lock-less linked list; the availability of a * rseq primitive allows us to implement pop without concerns over * ABA-type races. */struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, int *_cpu){ for (;;) { struct percpu_list_node *head; intptr_t *targetptr, expectnot, *load; long offset; int ret, cpu;
cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, targetptr, expectnot, offset, load, cpu); if (rseq_likely(!ret)) { if (_cpu) *_cpu = cpu; return head; } if (ret > 0) return NULL; /* Retry if rseq aborts. */ }}
/* * __percpu_list_pop is not safe against concurrent accesses. Should * only be used on lists that are not concurrently modified. */struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu){ struct percpu_list_node *node;
node = list->c[cpu].head; if (!node) return NULL; list->c[cpu].head = node->next; return node;}
void *test_percpu_list_thread(void *arg){ int i; struct percpu_list *list = (struct percpu_list *)arg;
if (rseq_register_current_thread()) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); }
for (i = 0; i < 100000; i++) { struct percpu_list_node *node;
node = this_cpu_list_pop(list, NULL); sched_yield(); /* encourage shuffling */ if (node) this_cpu_list_push(list, node, NULL); }
if (rseq_unregister_current_thread()) { fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); }
return NULL;}
/* Simultaneous modification to a per-cpu linked list from many threads. */void test_percpu_list(void){ int i, j; uint64_t sum = 0, expected_sum = 0; struct percpu_list list; pthread_t test_threads[200]; cpu_set_t allowed_cpus;
memset(&list, 0, sizeof(list));
/* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; for (j = 1; j <= 100; j++) { struct percpu_list_node *node;
expected_sum += j;
node = malloc(sizeof(*node)); assert(node); node->data = j; node->next = list.c[i].head; list.c[i].head = node; } }
for (i = 0; i < 200; i++) pthread_create(&test_threads[i], NULL, test_percpu_list_thread, &list);
for (i = 0; i < 200; i++) pthread_join(test_threads[i], NULL);
for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_list_node *node;
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue;
while ((node = __percpu_list_pop(&list, i))) { sum += node->data; free(node); } }
/* * All entries should now be accounted for (unless some external * actor is interfering with our allowed affinity while this * test is running). */ assert(sum == expected_sum);}
int main(int argc, char **argv){ if (rseq_register_current_thread()) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); goto error; } if (!rseq_validate_cpu_id()) { fprintf(stderr, "Error: cpu id getter unavailable\n"); goto error; } printf("spinlock\n"); test_percpu_spinlock(); printf("percpu_list\n"); test_percpu_list(); if (rseq_unregister_current_thread()) { fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); goto error; } return 0;
error: return -1;}