Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668// SPDX-License-Identifier: LGPL-2.1#define _GNU_SOURCE#include <assert.h>#include <linux/membarrier.h>#include <pthread.h>#include <sched.h>#include <stdatomic.h>#include <stdint.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <syscall.h>#include <unistd.h>#include <poll.h>#include <sys/types.h>#include <signal.h>#include <errno.h>#include <stddef.h>#include <stdbool.h>
static inline pid_t rseq_gettid(void){ return syscall(__NR_gettid);}
#define NR_INJECT 9static int loop_cnt[NR_INJECT + 1];
static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used));
static int opt_modulo, verbose;
static int opt_yield, opt_signal, opt_sleep, opt_disable_rseq, opt_threads = 200, opt_disable_mod = 0, opt_test = 's';static bool opt_rseq_legacy;static long long opt_reps = 5000;
static __thread __attribute__((tls_model("initial-exec")))unsigned int signals_delivered;
#ifndef BENCHMARK
static __thread __attribute__((tls_model("initial-exec"), unused))unsigned int yield_mod_cnt, nr_abort;
#define printf_verbose(fmt, ...) \ do { \ if (verbose) \ printf(fmt, ## __VA_ARGS__); \ } while (0)
#ifdef __i386__
#define INJECT_ASM_REG "eax"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \ "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ "jz 333f\n\t" \ "222:\n\t" \ "dec %%" INJECT_ASM_REG "\n\t" \ "jnz 222b\n\t" \ "333:\n\t"
#elif defined(__x86_64__)
#define INJECT_ASM_REG_P "rax"#define INJECT_ASM_REG "eax"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG_P \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \ "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \ "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ "jz 333f\n\t" \ "222:\n\t" \ "dec %%" INJECT_ASM_REG "\n\t" \ "jnz 222b\n\t" \ "333:\n\t"
#elif defined(__s390__)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ , [loop_cnt_2]"m"(loop_cnt[2]) \ , [loop_cnt_3]"m"(loop_cnt[3]) \ , [loop_cnt_4]"m"(loop_cnt[4]) \ , [loop_cnt_5]"m"(loop_cnt[5]) \ , [loop_cnt_6]"m"(loop_cnt[6])
#define INJECT_ASM_REG "r12"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \ "je 333f\n\t" \ "222:\n\t" \ "ahi %%" INJECT_ASM_REG ", -1\n\t" \ "jnz 222b\n\t" \ "333:\n\t"
#elif defined(__ARMEL__)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ , [loop_cnt_2]"m"(loop_cnt[2]) \ , [loop_cnt_3]"m"(loop_cnt[3]) \ , [loop_cnt_4]"m"(loop_cnt[4]) \ , [loop_cnt_5]"m"(loop_cnt[5]) \ , [loop_cnt_6]"m"(loop_cnt[6])
#define INJECT_ASM_REG "r4"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ "cmp " INJECT_ASM_REG ", #0\n\t" \ "beq 333f\n\t" \ "222:\n\t" \ "subs " INJECT_ASM_REG ", #1\n\t" \ "bne 222b\n\t" \ "333:\n\t"
#elif defined(__AARCH64EL__)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1] "Qo" (loop_cnt[1]) \ , [loop_cnt_2] "Qo" (loop_cnt[2]) \ , [loop_cnt_3] "Qo" (loop_cnt[3]) \ , [loop_cnt_4] "Qo" (loop_cnt[4]) \ , [loop_cnt_5] "Qo" (loop_cnt[5]) \ , [loop_cnt_6] "Qo" (loop_cnt[6])
#define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
#define RSEQ_INJECT_ASM(n) \ " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \ " cbz " INJECT_ASM_REG ", 333f\n" \ "222:\n" \ " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \ " cbnz " INJECT_ASM_REG ", 222b\n" \ "333:\n"
#elif defined(__PPC__)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ , [loop_cnt_2]"m"(loop_cnt[2]) \ , [loop_cnt_3]"m"(loop_cnt[3]) \ , [loop_cnt_4]"m"(loop_cnt[4]) \ , [loop_cnt_5]"m"(loop_cnt[5]) \ , [loop_cnt_6]"m"(loop_cnt[6])
#define INJECT_ASM_REG "r18"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \ "beq 333f\n\t" \ "222:\n\t" \ "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \ "bne 222b\n\t" \ "333:\n\t"
#elif defined(__mips__)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ , [loop_cnt_2]"m"(loop_cnt[2]) \ , [loop_cnt_3]"m"(loop_cnt[3]) \ , [loop_cnt_4]"m"(loop_cnt[4]) \ , [loop_cnt_5]"m"(loop_cnt[5]) \ , [loop_cnt_6]"m"(loop_cnt[6])
#define INJECT_ASM_REG "$5"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ "beqz " INJECT_ASM_REG ", 333f\n\t" \ "222:\n\t" \ "addiu " INJECT_ASM_REG ", -1\n\t" \ "bnez " INJECT_ASM_REG ", 222b\n\t" \ "333:\n\t"#elif defined(__riscv)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ , [loop_cnt_2]"m"(loop_cnt[2]) \ , [loop_cnt_3]"m"(loop_cnt[3]) \ , [loop_cnt_4]"m"(loop_cnt[4]) \ , [loop_cnt_5]"m"(loop_cnt[5]) \ , [loop_cnt_6]"m"(loop_cnt[6])
#define INJECT_ASM_REG "t1"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ "beqz " INJECT_ASM_REG ", 333f\n\t" \ "222:\n\t" \ "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \ "bnez " INJECT_ASM_REG ", 222b\n\t" \ "333:\n\t"#elif defined(__or1k__)
#define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ , [loop_cnt_2]"m"(loop_cnt[2]) \ , [loop_cnt_3]"m"(loop_cnt[3]) \ , [loop_cnt_4]"m"(loop_cnt[4]) \ , [loop_cnt_5]"m"(loop_cnt[5]) \ , [loop_cnt_6]"m"(loop_cnt[6])
#define INJECT_ASM_REG "r31"
#define RSEQ_INJECT_CLOBBER \ , INJECT_ASM_REG
#define RSEQ_INJECT_ASM(n) \ "l.lwz " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ "l.sfeqi " INJECT_ASM_REG ", 0\n\t" \ "l.bf 333f\n\t" \ " l.nop\n\t" \ "222:\n\t" \ "l.addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \ "l.sfeqi " INJECT_ASM_REG ", 0\n\t" \ "l.bf 222f\n\t" \ " l.nop\n\t" \ "333:\n\t"#else#error unsupported target#endif
#define RSEQ_INJECT_FAILED \ nr_abort++;
#define RSEQ_INJECT_C(n) \{ \ int loc_i, loc_nr_loops = loop_cnt[n]; \ \ for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \ rseq_barrier(); \ } \ if (loc_nr_loops == -1 && opt_modulo) { \ if (yield_mod_cnt == opt_modulo - 1) { \ if (opt_sleep > 0) \ poll(NULL, 0, opt_sleep); \ if (opt_yield) \ sched_yield(); \ if (opt_signal) \ raise(SIGUSR1); \ yield_mod_cnt = 0; \ } else { \ yield_mod_cnt++; \ } \ } \}
#define rseq_no_glibc true
#else
#define printf_verbose(fmt, ...)#define rseq_no_glibc false
#endif /* BENCHMARK */
#include "rseq.h"
static enum rseq_mo opt_mo = RSEQ_MO_RELAXED;
#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV#define TEST_MEMBARRIER
static int sys_membarrier(int cmd, int flags, int cpu_id){ return syscall(__NR_membarrier, cmd, flags, cpu_id);}#endif
#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID# define RSEQ_PERCPU RSEQ_PERCPU_MM_CIDstaticint get_current_cpu_id(void){ return rseq_current_mm_cid();}staticbool rseq_validate_cpu_id(void){ return rseq_mm_cid_available();}staticbool rseq_use_cpu_index(void){ return false; /* Use mm_cid */}# ifdef TEST_MEMBARRIER/* * Membarrier does not currently support targeting a mm_cid, so * issue the barrier on all cpus. */staticint rseq_membarrier_expedited(int cpu){ return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, 0, 0);}# endif /* TEST_MEMBARRIER */#else# define RSEQ_PERCPU RSEQ_PERCPU_CPU_IDstaticint get_current_cpu_id(void){ return rseq_cpu_start();}staticbool rseq_validate_cpu_id(void){ return rseq_current_cpu_raw() >= 0;}staticbool rseq_use_cpu_index(void){ return true; /* Use cpu_id as index. */}# ifdef TEST_MEMBARRIERstaticint rseq_membarrier_expedited(int cpu){ return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, MEMBARRIER_CMD_FLAG_CPU, cpu);}# endif /* TEST_MEMBARRIER */#endif
struct percpu_lock_entry { intptr_t v;} __attribute__((aligned(128)));
struct percpu_lock { struct percpu_lock_entry c[CPU_SETSIZE];};
struct test_data_entry { intptr_t count;} __attribute__((aligned(128)));
struct spinlock_test_data { struct percpu_lock lock; struct test_data_entry c[CPU_SETSIZE];};
struct spinlock_thread_test_data { struct spinlock_test_data *data; long long reps; int reg;};
struct inc_test_data { struct test_data_entry c[CPU_SETSIZE];};
struct inc_thread_test_data { struct inc_test_data *data; long long reps; int reg;};
struct percpu_list_node { intptr_t data; struct percpu_list_node *next;};
struct percpu_list_entry { struct percpu_list_node *head;} __attribute__((aligned(128)));
struct percpu_list { struct percpu_list_entry c[CPU_SETSIZE];};
#define BUFFER_ITEM_PER_CPU 100
struct percpu_buffer_node { intptr_t data;};
struct percpu_buffer_entry { intptr_t offset; intptr_t buflen; struct percpu_buffer_node **array;} __attribute__((aligned(128)));
struct percpu_buffer { struct percpu_buffer_entry c[CPU_SETSIZE];};
#define MEMCPY_BUFFER_ITEM_PER_CPU 100
struct percpu_memcpy_buffer_node { intptr_t data1; uint64_t data2;};
struct percpu_memcpy_buffer_entry { intptr_t offset; intptr_t buflen; struct percpu_memcpy_buffer_node *array;} __attribute__((aligned(128)));
struct percpu_memcpy_buffer { struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];};
/* A simple percpu spinlock. Grabs lock on current cpu. */static int rseq_this_cpu_lock(struct percpu_lock *lock){ int cpu;
for (;;) { int ret;
cpu = get_current_cpu_id(); if (cpu < 0) { fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n", getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu); abort(); } ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ } /* * Acquire semantic when taking lock after control dependency. * Matches rseq_smp_store_release(). */ rseq_smp_acquire__after_ctrl_dep(); return cpu;}
static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu){ assert(lock->c[cpu].v == 1); /* * Release lock, with release semantic. Matches * rseq_smp_acquire__after_ctrl_dep(). */ rseq_smp_store_release(&lock->c[cpu].v, 0);}
void *test_percpu_spinlock_thread(void *arg){ struct spinlock_thread_test_data *thread_data = arg; struct spinlock_test_data *data = thread_data->data; long long i, reps;
if (!opt_disable_rseq && thread_data->reg && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = thread_data->reps; for (i = 0; i < reps; i++) { int cpu = rseq_this_cpu_lock(&data->lock); data->c[cpu].count++; rseq_percpu_unlock(&data->lock, cpu);#ifndef BENCHMARK if (i != 0 && !(i % (reps / 10))) printf_verbose("tid %d: count %lld\n", (int) rseq_gettid(), i);#endif } printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", (int) rseq_gettid(), nr_abort, signals_delivered); if (!opt_disable_rseq && thread_data->reg && rseq_unregister_current_thread()) abort(); return NULL;}
/* * A simple test which implements a sharded counter using a per-cpu * lock. Obviously real applications might prefer to simply use a * per-cpu increment; however, this is reasonable for a test and the * lock can be extended to synchronize more complicated operations. */void test_percpu_spinlock(void){ const int num_threads = opt_threads; int i, ret; uint64_t sum; pthread_t test_threads[num_threads]; struct spinlock_test_data data; struct spinlock_thread_test_data thread_data[num_threads];
memset(&data, 0, sizeof(data)); for (i = 0; i < num_threads; i++) { thread_data[i].reps = opt_reps; if (opt_disable_mod <= 0 || (i % opt_disable_mod)) thread_data[i].reg = 1; else thread_data[i].reg = 0; thread_data[i].data = &data; ret = pthread_create(&test_threads[i], NULL, test_percpu_spinlock_thread, &thread_data[i]); if (ret) { errno = ret; perror("pthread_create"); abort(); } }
for (i = 0; i < num_threads; i++) { ret = pthread_join(test_threads[i], NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); } }
sum = 0; for (i = 0; i < CPU_SETSIZE; i++) sum += data.c[i].count;
assert(sum == (uint64_t)opt_reps * num_threads);}
void *test_percpu_inc_thread(void *arg){ struct inc_thread_test_data *thread_data = arg; struct inc_test_data *data = thread_data->data; long long i, reps;
if (!opt_disable_rseq && thread_data->reg && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = thread_data->reps; for (i = 0; i < reps; i++) { int ret;
do { int cpu;
cpu = get_current_cpu_id(); ret = rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, &data->c[cpu].count, 1, cpu); } while (rseq_unlikely(ret));#ifndef BENCHMARK if (i != 0 && !(i % (reps / 10))) printf_verbose("tid %d: count %lld\n", (int) rseq_gettid(), i);#endif } printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", (int) rseq_gettid(), nr_abort, signals_delivered); if (!opt_disable_rseq && thread_data->reg && rseq_unregister_current_thread()) abort(); return NULL;}
void test_percpu_inc(void){ const int num_threads = opt_threads; int i, ret; uint64_t sum; pthread_t test_threads[num_threads]; struct inc_test_data data; struct inc_thread_test_data thread_data[num_threads];
memset(&data, 0, sizeof(data)); for (i = 0; i < num_threads; i++) { thread_data[i].reps = opt_reps; if (opt_disable_mod <= 0 || (i % opt_disable_mod)) thread_data[i].reg = 1; else thread_data[i].reg = 0; thread_data[i].data = &data; ret = pthread_create(&test_threads[i], NULL, test_percpu_inc_thread, &thread_data[i]); if (ret) { errno = ret; perror("pthread_create"); abort(); } }
for (i = 0; i < num_threads; i++) { ret = pthread_join(test_threads[i], NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); } }
sum = 0; for (i = 0; i < CPU_SETSIZE; i++) sum += data.c[i].count;
assert(sum == (uint64_t)opt_reps * num_threads);}
void this_cpu_list_push(struct percpu_list *list, struct percpu_list_node *node, int *_cpu){ int cpu;
for (;;) { intptr_t *targetptr, newval, expect; int ret;
cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ } if (_cpu) *_cpu = cpu;}
/* * Unlike a traditional lock-less linked list; the availability of a * rseq primitive allows us to implement pop without concerns over * ABA-type races. */struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, int *_cpu){ struct percpu_list_node *node = NULL; int cpu;
for (;;) { struct percpu_list_node *head; intptr_t *targetptr, expectnot, *load; long offset; int ret;
cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, targetptr, expectnot, offset, load, cpu); if (rseq_likely(!ret)) { node = head; break; } if (ret > 0) break; /* Retry if rseq aborts. */ } if (_cpu) *_cpu = cpu; return node;}
/* * __percpu_list_pop is not safe against concurrent accesses. Should * only be used on lists that are not concurrently modified. */struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu){ struct percpu_list_node *node;
node = list->c[cpu].head; if (!node) return NULL; list->c[cpu].head = node->next; return node;}
void *test_percpu_list_thread(void *arg){ long long i, reps; struct percpu_list *list = (struct percpu_list *)arg;
if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort();
reps = opt_reps; for (i = 0; i < reps; i++) { struct percpu_list_node *node;
node = this_cpu_list_pop(list, NULL); if (opt_yield) sched_yield(); /* encourage shuffling */ if (node) this_cpu_list_push(list, node, NULL); }
printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", (int) rseq_gettid(), nr_abort, signals_delivered); if (!opt_disable_rseq && rseq_unregister_current_thread()) abort();
return NULL;}
/* Simultaneous modification to a per-cpu linked list from many threads. */void test_percpu_list(void){ const int num_threads = opt_threads; int i, j, ret; uint64_t sum = 0, expected_sum = 0; struct percpu_list list; pthread_t test_threads[num_threads]; cpu_set_t allowed_cpus;
memset(&list, 0, sizeof(list));
/* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; for (j = 1; j <= 100; j++) { struct percpu_list_node *node;
expected_sum += j;
node = malloc(sizeof(*node)); assert(node); node->data = j; node->next = list.c[i].head; list.c[i].head = node; } }
for (i = 0; i < num_threads; i++) { ret = pthread_create(&test_threads[i], NULL, test_percpu_list_thread, &list); if (ret) { errno = ret; perror("pthread_create"); abort(); } }
for (i = 0; i < num_threads; i++) { ret = pthread_join(test_threads[i], NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); } }
for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_list_node *node;
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue;
while ((node = __percpu_list_pop(&list, i))) { sum += node->data; free(node); } }
/* * All entries should now be accounted for (unless some external * actor is interfering with our allowed affinity while this * test is running). */ assert(sum == expected_sum);}
bool this_cpu_buffer_push(struct percpu_buffer *buffer, struct percpu_buffer_node *node, int *_cpu){ bool result = false; int cpu;
for (;;) { intptr_t *targetptr_spec, newval_spec; intptr_t *targetptr_final, newval_final; intptr_t offset; int ret;
cpu = get_current_cpu_id(); offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) break; newval_spec = (intptr_t)node; targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; ret = rseq_cmpeqv_trystorev_storev(opt_mo, RSEQ_PERCPU, targetptr_final, offset, targetptr_spec, newval_spec, newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; } /* Retry if comparison fails or rseq aborts. */ } if (_cpu) *_cpu = cpu; return result;}
struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, int *_cpu){ struct percpu_buffer_node *head; int cpu;
for (;;) { intptr_t *targetptr, newval; intptr_t offset; int ret;
cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) { head = NULL; break; } head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); newval = offset - 1; targetptr = (intptr_t *)&buffer->c[cpu].offset; ret = rseq_cmpeqv_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, targetptr, offset, (intptr_t *)&buffer->c[cpu].array[offset - 1], (intptr_t)head, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ } if (_cpu) *_cpu = cpu; return head;}
/* * __percpu_buffer_pop is not safe against concurrent accesses. Should * only be used on buffers that are not concurrently modified. */struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer, int cpu){ struct percpu_buffer_node *head; intptr_t offset;
offset = buffer->c[cpu].offset; if (offset == 0) return NULL; head = buffer->c[cpu].array[offset - 1]; buffer->c[cpu].offset = offset - 1; return head;}
void *test_percpu_buffer_thread(void *arg){ long long i, reps; struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort();
reps = opt_reps; for (i = 0; i < reps; i++) { struct percpu_buffer_node *node;
node = this_cpu_buffer_pop(buffer, NULL); if (opt_yield) sched_yield(); /* encourage shuffling */ if (node) { if (!this_cpu_buffer_push(buffer, node, NULL)) { /* Should increase buffer size. */ abort(); } } }
printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", (int) rseq_gettid(), nr_abort, signals_delivered); if (!opt_disable_rseq && rseq_unregister_current_thread()) abort();
return NULL;}
/* Simultaneous modification to a per-cpu buffer from many threads. */void test_percpu_buffer(void){ const int num_threads = opt_threads; int i, j, ret; uint64_t sum = 0, expected_sum = 0; struct percpu_buffer buffer; pthread_t test_threads[num_threads]; cpu_set_t allowed_cpus;
memset(&buffer, 0, sizeof(buffer));
/* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; /* Worse-case is every item in same CPU. */ buffer.c[i].array = malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * BUFFER_ITEM_PER_CPU); assert(buffer.c[i].array); buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU; for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) { struct percpu_buffer_node *node;
expected_sum += j;
/* * We could theoretically put the word-sized * "data" directly in the buffer. However, we * want to model objects that would not fit * within a single word, so allocate an object * for each node. */ node = malloc(sizeof(*node)); assert(node); node->data = j; buffer.c[i].array[j - 1] = node; buffer.c[i].offset++; } }
for (i = 0; i < num_threads; i++) { ret = pthread_create(&test_threads[i], NULL, test_percpu_buffer_thread, &buffer); if (ret) { errno = ret; perror("pthread_create"); abort(); } }
for (i = 0; i < num_threads; i++) { ret = pthread_join(test_threads[i], NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); } }
for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_buffer_node *node;
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue;
while ((node = __percpu_buffer_pop(&buffer, i))) { sum += node->data; free(node); } free(buffer.c[i].array); }
/* * All entries should now be accounted for (unless some external * actor is interfering with our allowed affinity while this * test is running). */ assert(sum == expected_sum);}
bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, struct percpu_memcpy_buffer_node item, int *_cpu){ bool result = false; int cpu;
for (;;) { intptr_t *targetptr_final, newval_final, offset; char *destptr, *srcptr; size_t copylen; int ret;
cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) break; destptr = (char *)&buffer->c[cpu].array[offset]; srcptr = (char *)&item; /* copylen must be <= 4kB. */ copylen = sizeof(item); newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; ret = rseq_cmpeqv_trymemcpy_storev( opt_mo, RSEQ_PERCPU, targetptr_final, offset, destptr, srcptr, copylen, newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; } /* Retry if comparison fails or rseq aborts. */ } if (_cpu) *_cpu = cpu; return result;}
bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, struct percpu_memcpy_buffer_node *item, int *_cpu){ bool result = false; int cpu;
for (;;) { intptr_t *targetptr_final, newval_final, offset; char *destptr, *srcptr; size_t copylen; int ret;
cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) break; destptr = (char *)item; srcptr = (char *)&buffer->c[cpu].array[offset - 1]; /* copylen must be <= 4kB. */ copylen = sizeof(*item); newval_final = offset - 1; targetptr_final = &buffer->c[cpu].offset; ret = rseq_cmpeqv_trymemcpy_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, targetptr_final, offset, destptr, srcptr, copylen, newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; } /* Retry if comparison fails or rseq aborts. */ } if (_cpu) *_cpu = cpu; return result;}
/* * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should * only be used on buffers that are not concurrently modified. */bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, struct percpu_memcpy_buffer_node *item, int cpu){ intptr_t offset;
offset = buffer->c[cpu].offset; if (offset == 0) return false; memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); buffer->c[cpu].offset = offset - 1; return true;}
void *test_percpu_memcpy_buffer_thread(void *arg){ long long i, reps; struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort();
reps = opt_reps; for (i = 0; i < reps; i++) { struct percpu_memcpy_buffer_node item; bool result;
result = this_cpu_memcpy_buffer_pop(buffer, &item, NULL); if (opt_yield) sched_yield(); /* encourage shuffling */ if (result) { if (!this_cpu_memcpy_buffer_push(buffer, item, NULL)) { /* Should increase buffer size. */ abort(); } } }
printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n", (int) rseq_gettid(), nr_abort, signals_delivered); if (!opt_disable_rseq && rseq_unregister_current_thread()) abort();
return NULL;}
/* Simultaneous modification to a per-cpu buffer from many threads. */void test_percpu_memcpy_buffer(void){ const int num_threads = opt_threads; int i, j, ret; uint64_t sum = 0, expected_sum = 0; struct percpu_memcpy_buffer buffer; pthread_t test_threads[num_threads]; cpu_set_t allowed_cpus;
memset(&buffer, 0, sizeof(buffer));
/* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; /* Worse-case is every item in same CPU. */ buffer.c[i].array = malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU); assert(buffer.c[i].array); buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU; for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) { expected_sum += 2 * j + 1;
/* * We could theoretically put the word-sized * "data" directly in the buffer. However, we * want to model objects that would not fit * within a single word, so allocate an object * for each node. */ buffer.c[i].array[j - 1].data1 = j; buffer.c[i].array[j - 1].data2 = j + 1; buffer.c[i].offset++; } }
for (i = 0; i < num_threads; i++) { ret = pthread_create(&test_threads[i], NULL, test_percpu_memcpy_buffer_thread, &buffer); if (ret) { errno = ret; perror("pthread_create"); abort(); } }
for (i = 0; i < num_threads; i++) { ret = pthread_join(test_threads[i], NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); } }
for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_memcpy_buffer_node item;
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue;
while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) { sum += item.data1; sum += item.data2; } free(buffer.c[i].array); }
/* * All entries should now be accounted for (unless some external * actor is interfering with our allowed affinity while this * test is running). */ assert(sum == expected_sum);}
static void test_signal_interrupt_handler(int signo){ signals_delivered++;}
static int set_signal_handler(void){ int ret = 0; struct sigaction sa; sigset_t sigset;
ret = sigemptyset(&sigset); if (ret < 0) { perror("sigemptyset"); return ret; }
sa.sa_handler = test_signal_interrupt_handler; sa.sa_mask = sigset; sa.sa_flags = 0; ret = sigaction(SIGUSR1, &sa, NULL); if (ret < 0) { perror("sigaction"); return ret; }
printf_verbose("Signal handler set for SIGUSR1\n");
return ret;}
/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */#ifdef TEST_MEMBARRIERstruct test_membarrier_thread_args { int stop; intptr_t percpu_list_ptr;};
/* Worker threads modify data in their "active" percpu lists. */void *test_membarrier_worker_thread(void *arg){ struct test_membarrier_thread_args *args = (struct test_membarrier_thread_args *)arg; const int iters = opt_reps; int i;
if (__rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); }
/* Wait for initialization. */ while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {}
for (i = 0; i < iters; ++i) { int ret;
do { int cpu = get_current_cpu_id();
ret = rseq_offset_deref_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, &args->percpu_list_ptr, sizeof(struct percpu_list_entry) * cpu, 1, cpu); } while (rseq_unlikely(ret)); }
if (rseq_unregister_current_thread()) { fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); } return NULL;}
void test_membarrier_init_percpu_list(struct percpu_list *list){ int i;
memset(list, 0, sizeof(*list)); for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_list_node *node;
node = malloc(sizeof(*node)); assert(node); node->data = 0; node->next = NULL; list->c[i].head = node; }}
void test_membarrier_free_percpu_list(struct percpu_list *list){ int i;
for (i = 0; i < CPU_SETSIZE; i++) free(list->c[i].head);}
/* * The manager thread swaps per-cpu lists that worker threads see, * and validates that there are no unexpected modifications. */void *test_membarrier_manager_thread(void *arg){ struct test_membarrier_thread_args *args = (struct test_membarrier_thread_args *)arg; struct percpu_list list_a, list_b; intptr_t expect_a = 0, expect_b = 0; int cpu_a = 0, cpu_b = 0;
if (__rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); }
/* Init lists. */ test_membarrier_init_percpu_list(&list_a); test_membarrier_init_percpu_list(&list_b);
__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);
while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) { /* list_a is "active". */ cpu_a = rand() % CPU_SETSIZE; /* * As list_b is "inactive", we should never see changes * to list_b. */ if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) { fprintf(stderr, "Membarrier test failed\n"); abort(); }
/* Make list_b "active". */ __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE); if (rseq_membarrier_expedited(cpu_a) && errno != ENXIO /* missing CPU */) { perror("sys_membarrier"); abort(); } /* * Cpu A should now only modify list_b, so the values * in list_a should be stable. */ expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE);
cpu_b = rand() % CPU_SETSIZE; /* * As list_a is "inactive", we should never see changes * to list_a. */ if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) { fprintf(stderr, "Membarrier test failed\n"); abort(); }
/* Make list_a "active". */ __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); if (rseq_membarrier_expedited(cpu_b) && errno != ENXIO /* missing CPU*/) { perror("sys_membarrier"); abort(); } /* Remember a value from list_b. */ expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE); }
test_membarrier_free_percpu_list(&list_a); test_membarrier_free_percpu_list(&list_b);
if (rseq_unregister_current_thread()) { fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); } return NULL;}
void test_membarrier(void){ const int num_threads = opt_threads; struct test_membarrier_thread_args thread_args; pthread_t worker_threads[num_threads]; pthread_t manager_thread; int i, ret;
if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0)) { perror("sys_membarrier"); abort(); }
thread_args.stop = 0; thread_args.percpu_list_ptr = 0; ret = pthread_create(&manager_thread, NULL, test_membarrier_manager_thread, &thread_args); if (ret) { errno = ret; perror("pthread_create"); abort(); }
for (i = 0; i < num_threads; i++) { ret = pthread_create(&worker_threads[i], NULL, test_membarrier_worker_thread, &thread_args); if (ret) { errno = ret; perror("pthread_create"); abort(); } }
for (i = 0; i < num_threads; i++) { ret = pthread_join(worker_threads[i], NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); } }
__atomic_store_n(&thread_args.stop, 1, __ATOMIC_RELEASE); ret = pthread_join(manager_thread, NULL); if (ret) { errno = ret; perror("pthread_join"); abort(); }}#else /* TEST_MEMBARRIER */void test_membarrier(void){ fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. " "Skipping membarrier test.\n");}#endif
static void show_usage(int argc, char **argv){ printf("Usage : %s <OPTIONS>\n", argv[0]); printf("OPTIONS:\n"); printf(" [-1 loops] Number of loops for delay injection 1\n"); printf(" [-2 loops] Number of loops for delay injection 2\n"); printf(" [-3 loops] Number of loops for delay injection 3\n"); printf(" [-4 loops] Number of loops for delay injection 4\n"); printf(" [-5 loops] Number of loops for delay injection 5\n"); printf(" [-6 loops] Number of loops for delay injection 6\n"); printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); printf(" [-y] Yield\n"); printf(" [-k] Kill thread with signal\n"); printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); printf(" [-t N] Number of threads (default 200)\n"); printf(" [-r N] Number of repetitions per thread (default 5000)\n"); printf(" [-d] Disable rseq system call (no initialization)\n"); printf(" [-D M] Disable rseq for each M threads\n"); printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n"); printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); printf(" [-O] Test with optimized RSEQ\n"); printf(" [-v] Verbose output.\n"); printf(" [-h] Show this help.\n"); printf("\n");}
int main(int argc, char **argv){ int i;
for (i = 1; i < argc; i++) { if (argv[i][0] != '-') continue; switch (argv[i][1]) { case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': if (argc < i + 2) { show_usage(argc, argv); goto error; } loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); i++; break; case 'm': if (argc < i + 2) { show_usage(argc, argv); goto error; } opt_modulo = atol(argv[i + 1]); if (opt_modulo < 0) { show_usage(argc, argv); goto error; } i++; break; case 's': if (argc < i + 2) { show_usage(argc, argv); goto error; } opt_sleep = atol(argv[i + 1]); if (opt_sleep < 0) { show_usage(argc, argv); goto error; } i++; break; case 'y': opt_yield = 1; break; case 'k': opt_signal = 1; break; case 'd': opt_disable_rseq = 1; break; case 'D': if (argc < i + 2) { show_usage(argc, argv); goto error; } opt_disable_mod = atol(argv[i + 1]); if (opt_disable_mod < 0) { show_usage(argc, argv); goto error; } i++; break; case 't': if (argc < i + 2) { show_usage(argc, argv); goto error; } opt_threads = atol(argv[i + 1]); if (opt_threads < 0) { show_usage(argc, argv); goto error; } i++; break; case 'r': if (argc < i + 2) { show_usage(argc, argv); goto error; } opt_reps = atoll(argv[i + 1]); if (opt_reps < 0) { show_usage(argc, argv); goto error; } i++; break; case 'h': show_usage(argc, argv); goto end; case 'T': if (argc < i + 2) { show_usage(argc, argv); goto error; } opt_test = *argv[i + 1]; switch (opt_test) { case 's': case 'l': case 'i': case 'b': case 'm': case 'r': break; default: show_usage(argc, argv); goto error; } i++; break; case 'v': verbose = 1; break; case 'M': opt_mo = RSEQ_MO_RELEASE; break; case 'L': opt_rseq_legacy = true; break; default: show_usage(argc, argv); goto error; } }
loop_cnt_1 = loop_cnt[1]; loop_cnt_2 = loop_cnt[2]; loop_cnt_3 = loop_cnt[3]; loop_cnt_4 = loop_cnt[4]; loop_cnt_5 = loop_cnt[5]; loop_cnt_6 = loop_cnt[6];
if (set_signal_handler()) goto error;
if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) goto error; if (!opt_disable_rseq && !rseq_validate_cpu_id()) { fprintf(stderr, "Error: cpu id getter unavailable\n"); goto error; } switch (opt_test) { case 's': printf_verbose("spinlock\n"); test_percpu_spinlock(); break; case 'l': printf_verbose("linked list\n"); test_percpu_list(); break; case 'b': printf_verbose("buffer\n"); test_percpu_buffer(); break; case 'm': printf_verbose("memcpy buffer\n"); test_percpu_memcpy_buffer(); break; case 'i': printf_verbose("counter increment\n"); test_percpu_inc(); break; case 'r': printf_verbose("membarrier\n"); test_membarrier(); break; } if (!opt_disable_rseq && rseq_unregister_current_thread()) abort();end: return 0;
error: return -1;}